net/ipv6/ila/ila_xlat.c

   1 #include <linux/jhash.h>
   2 #include <linux/netfilter.h>
   3 #include <linux/rcupdate.h>
   4 #include <linux/rhashtable.h>
   5 #include <linux/vmalloc.h>
   6 #include <net/genetlink.h>
   7 #include <net/ila.h>
   8 #include <net/netns/generic.h>
   9 #include <uapi/linux/genetlink.h>
  10 #include "ila.h"
  11
  12 struct ila_xlat_params {
  13         struct ila_params ip;
  14         int ifindex;
  15 };
  16
  17 struct ila_map {
  18         struct ila_xlat_params xp;
  19         struct rhash_head node;
  20         struct ila_map __rcu *next;
  21         struct rcu_head rcu;
  22 };
  23
  24 static unsigned int ila_net_id;
  25
  26 struct ila_net {
  27         struct rhashtable rhash_table;
  28         spinlock_t *locks; /* Bucket locks for entry manipulation */
  29         unsigned int locks_mask;
  30         bool hooks_registered;
  31 };
  32
  33 #define LOCKS_PER_CPU 10
  34
  35 static int alloc_ila_locks(struct ila_net *ilan)
  36 {
  37         unsigned int i, size;
  38         unsigned int nr_pcpus = num_possible_cpus();
  39
  40         nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
  41         size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);
  42
  43         if (sizeof(spinlock_t) != 0) {
  44 #ifdef CONFIG_NUMA
  45                 if (size * sizeof(spinlock_t) > PAGE_SIZE)
  46                         ilan->locks = vmalloc(size * sizeof(spinlock_t));
  47                 else
  48 #endif
  49                 ilan->locks = kmalloc_array(size, sizeof(spinlock_t),
  50                                             GFP_KERNEL);
  51                 if (!ilan->locks)
  52                         return -ENOMEM;
  53                 for (i = 0; i < size; i++)
  54                         spin_lock_init(&ilan->locks[i]);
  55         }
  56         ilan->locks_mask = size - 1;
  57
  58         return 0;
  59 }
  60
  61 static u32 hashrnd __read_mostly;
  62 static __always_inline void __ila_hash_secret_init(void)
  63 {
  64         net_get_random_once(&hashrnd, sizeof(hashrnd));
  65 }
  66
  67 static inline u32 ila_locator_hash(struct ila_locator loc)
  68 {
  69         u32 *v = (u32 *)loc.v32;
  70
  71         __ila_hash_secret_init();
  72         return jhash_2words(v[0], v[1], hashrnd);
  73 }
  74
  75 static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
  76                                        struct ila_locator loc)
  77 {
  78         return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
  79 }
  80
  81 static inline int ila_cmp_wildcards(struct ila_map *ila,
  82                                     struct ila_addr *iaddr, int ifindex)
  83 {
  84         return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
  85 }
  86
  87 static inline int ila_cmp_params(struct ila_map *ila,
  88                                  struct ila_xlat_params *xp)
  89 {
  90         return (ila->xp.ifindex != xp->ifindex);
  91 }
  92
  93 static int ila_cmpfn(struct rhashtable_compare_arg *arg,
  94                      const void *obj)
  95 {
  96         const struct ila_map *ila = obj;
  97
  98         return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key);
  99 }
 100
 101 static inline int ila_order(struct ila_map *ila)
 102 {
 103         int score = 0;
 104
 105         if (ila->xp.ifindex)
 106                 score += 1 << 1;
 107
 108         return score;
 109 }
 110
 111 static const struct rhashtable_params rht_params = {
 112         .nelem_hint = 1024,
 113         .head_offset = offsetof(struct ila_map, node),
 114         .key_offset = offsetof(struct ila_map, xp.ip.locator_match),
 115         .key_len = sizeof(u64), /* identifier */
 116         .max_size = 1048576,
 117         .min_size = 256,
 118         .automatic_shrinking = true,
 119         .obj_cmpfn = ila_cmpfn,
 120 };
 121
 122 static struct genl_family ila_nl_family = {
 123         .id             = GENL_ID_GENERATE,
 124         .hdrsize        = 0,
 125         .name           = ILA_GENL_NAME,
 126         .version        = ILA_GENL_VERSION,
 127         .maxattr        = ILA_ATTR_MAX,
 128         .netnsok        = true,
 129         .parallel_ops   = true,
 130 };
 131
 132 static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
 133         [ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
 134         [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
 135         [ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
 136         [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
 137 };
 138
 139 static int parse_nl_config(struct genl_info *info,
 140                            struct ila_xlat_params *xp)
 141 {
 142         memset(xp, 0, sizeof(*xp));
 143
 144         if (info->attrs[ILA_ATTR_LOCATOR])
 145                 xp->ip.locator.v64 = (__force __be64)nla_get_u64(
 146                         info->attrs[ILA_ATTR_LOCATOR]);
 147
 148         if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
 149                 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
 150                         info->attrs[ILA_ATTR_LOCATOR_MATCH]);
 151
 152         if (info->attrs[ILA_ATTR_CSUM_MODE])
 153                 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);
 154
 155         if (info->attrs[ILA_ATTR_IFINDEX])
 156                 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);
 157
 158         return 0;
 159 }
 160
 161 /* Must be called with rcu readlock */
 162 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
 163                                                    int ifindex,
 164                                                    struct ila_net *ilan)
 165 {
 166         struct ila_map *ila;
 167
 168         ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
 169                                      rht_params);
 170         while (ila) {
 171                 if (!ila_cmp_wildcards(ila, iaddr, ifindex))
 172                         return ila;
 173                 ila = rcu_access_pointer(ila->next);
 174         }
 175
 176         return NULL;
 177 }
 178
 179 /* Must be called with rcu readlock */
 180 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
 181                                                    struct ila_net *ilan)
 182 {
 183         struct ila_map *ila;
 184
 185         ila = rhashtable_lookup_fast(&ilan->rhash_table,
 186                                      &xp->ip.locator_match,
 187                                      rht_params);
 188         while (ila) {
 189                 if (!ila_cmp_params(ila, xp))
 190                         return ila;
 191                 ila = rcu_access_pointer(ila->next);
 192         }
 193
 194         return NULL;
 195 }
 196
 197 static inline void ila_release(struct ila_map *ila)
 198 {
 199         kfree_rcu(ila, rcu);
 200 }
 201
 202 static void ila_free_cb(void *ptr, void *arg)
 203 {
 204         struct ila_map *ila = (struct ila_map *)ptr, *next;
 205
 206         /* Assume rcu_readlock held */
 207         while (ila) {
 208                 next = rcu_access_pointer(ila->next);
 209                 ila_release(ila);
 210                 ila = next;
 211         }
 212 }
 213
 214 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral);
 215
 216 static unsigned int
 217 ila_nf_input(void *priv,
 218              struct sk_buff *skb,
 219              const struct nf_hook_state *state)
 220 {
 221         ila_xlat_addr(skb, false);
 222         return NF_ACCEPT;
 223 }
 224
 225 static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = {
 226         {
 227                 .hook = ila_nf_input,
 228                 .pf = NFPROTO_IPV6,
 229                 .hooknum = NF_INET_PRE_ROUTING,
 230                 .priority = -1,
 231         },
 232 };
 233
 234 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
 235 {
 236         struct ila_net *ilan = net_generic(net, ila_net_id);
 237         struct ila_map *ila, *head;
 238         spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
 239         int err = 0, order;
 240
 241         if (!ilan->hooks_registered) {
 242                 /* We defer registering net hooks in the namespace until the
 243                  * first mapping is added.
 244                  */
 245                 err = nf_register_net_hooks(net, ila_nf_hook_ops,
 246                                             ARRAY_SIZE(ila_nf_hook_ops));
 247                 if (err)
 248                         return err;
 249
 250                 ilan->hooks_registered = true;
 251         }
 252
 253         ila = kzalloc(sizeof(*ila), GFP_KERNEL);
 254         if (!ila)
 255                 return -ENOMEM;
 256
 257         ila_init_saved_csum(&xp->ip);
 258
 259         ila->xp = *xp;
 260
 261         order = ila_order(ila);
 262
 263         spin_lock(lock);
 264
 265         head = rhashtable_lookup_fast(&ilan->rhash_table,
 266                                       &xp->ip.locator_match,
 267                                       rht_params);
 268         if (!head) {
 269                 /* New entry for the rhash_table */
 270                 err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
 271                                                     &ila->node, rht_params);
 272         } else {
 273                 struct ila_map *tila = head, *prev = NULL;
 274
 275                 do {
 276                         if (!ila_cmp_params(tila, xp)) {
 277                                 err = -EEXIST;
 278                                 goto out;
 279                         }
 280
 281                         if (order > ila_order(tila))
 282                                 break;
 283
 284                         prev = tila;
 285                         tila = rcu_dereference_protected(tila->next,
 286                                 lockdep_is_held(lock));
 287                 } while (tila);
 288
 289                 if (prev) {
 290                         /* Insert in sub list of head */
 291                         RCU_INIT_POINTER(ila->next, tila);
 292                         rcu_assign_pointer(prev->next, ila);
 293                 } else {
 294                         /* Make this ila new head */
 295                         RCU_INIT_POINTER(ila->next, head);
 296                         err = rhashtable_replace_fast(&ilan->rhash_table,
 297                                                       &head->node,
 298                                                       &ila->node, rht_params);
 299                         if (err)
 300                                 goto out;
 301                 }
 302         }
 303
 304 out:
 305         spin_unlock(lock);
 306
 307         if (err)
 308                 kfree(ila);
 309
 310         return err;
 311 }
 312
 313 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
 314 {
 315         struct ila_net *ilan = net_generic(net, ila_net_id);
 316         struct ila_map *ila, *head, *prev;
 317         spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
 318         int err = -ENOENT;
 319
 320         spin_lock(lock);
 321
 322         head = rhashtable_lookup_fast(&ilan->rhash_table,
 323                                       &xp->ip.locator_match, rht_params);
 324         ila = head;
 325
 326         prev = NULL;
 327
 328         while (ila) {
 329                 if (ila_cmp_params(ila, xp)) {
 330                         prev = ila;
 331                         ila = rcu_dereference_protected(ila->next,
 332                                                         lockdep_is_held(lock));
 333                         continue;
 334                 }
 335
 336                 err = 0;
 337
 338                 if (prev) {
 339                         /* Not head, just delete from list */
 340                         rcu_assign_pointer(prev->next, ila->next);
 341                 } else {
 342                         /* It is the head. If there is something in the
 343                          * sublist we need to make a new head.
 344                          */
 345                         head = rcu_dereference_protected(ila->next,
 346                                                          lockdep_is_held(lock));
 347                         if (head) {
 348                                 /* Put first entry in the sublist into the
 349                                  * table
 350                                  */
 351                                 err = rhashtable_replace_fast(
 352                                         &ilan->rhash_table, &ila->node,
 353                                         &head->node, rht_params);
 354                                 if (err)
 355                                         goto out;
 356                         } else {
 357                                 /* Entry no longer used */
 358                                 err = rhashtable_remove_fast(&ilan->rhash_table,
 359                                                              &ila->node,
 360                                                              rht_params);
 361                         }
 362                 }
 363
 364                 ila_release(ila);
 365
 366                 break;
 367         }
 368
 369 out:
 370         spin_unlock(lock);
 371
 372         return err;
 373 }
 374
 375 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
 376 {
 377         struct net *net = genl_info_net(info);
 378         struct ila_xlat_params p;
 379         int err;
 380
 381         err = parse_nl_config(info, &p);
 382         if (err)
 383                 return err;
 384
 385         return ila_add_mapping(net, &p);
 386 }
 387
 388 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
 389 {
 390         struct net *net = genl_info_net(info);
 391         struct ila_xlat_params xp;
 392         int err;
 393
 394         err = parse_nl_config(info, &xp);
 395         if (err)
 396                 return err;
 397
 398         ila_del_mapping(net, &xp);
 399
 400         return 0;
 401 }
 402
 403 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
 404 {
 405         if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
 406                               (__force u64)ila->xp.ip.locator.v64,
 407                               ILA_ATTR_PAD) ||
 408             nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
 409                               (__force u64)ila->xp.ip.locator_match.v64,
 410                               ILA_ATTR_PAD) ||
 411             nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) ||
 412             nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode))
 413                 return -1;
 414
 415         return 0;
 416 }
 417
 418 static int ila_dump_info(struct ila_map *ila,
 419                          u32 portid, u32 seq, u32 flags,
 420                          struct sk_buff *skb, u8 cmd)
 421 {
 422         void *hdr;
 423
 424         hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
 425         if (!hdr)
 426                 return -ENOMEM;
 427
 428         if (ila_fill_info(ila, skb) < 0)
 429                 goto nla_put_failure;
 430
 431         genlmsg_end(skb, hdr);
 432         return 0;
 433
 434 nla_put_failure:
 435         genlmsg_cancel(skb, hdr);
 436         return -EMSGSIZE;
 437 }
 438
 439 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
 440 {
 441         struct net *net = genl_info_net(info);
 442         struct ila_net *ilan = net_generic(net, ila_net_id);
 443         struct sk_buff *msg;
 444         struct ila_xlat_params xp;
 445         struct ila_map *ila;
 446         int ret;
 447
 448         ret = parse_nl_config(info, &xp);
 449         if (ret)
 450                 return ret;
 451
 452         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 453         if (!msg)
 454                 return -ENOMEM;
 455
 456         rcu_read_lock();
 457
 458         ila = ila_lookup_by_params(&xp, ilan);
 459         if (ila) {
 460                 ret = ila_dump_info(ila,
 461                                     info->snd_portid,
 462                                     info->snd_seq, 0, msg,
 463                                     info->genlhdr->cmd);
 464         }
 465
 466         rcu_read_unlock();
 467
 468         if (ret < 0)
 469                 goto out_free;
 470
 471         return genlmsg_reply(msg, info);
 472
 473 out_free:
 474         nlmsg_free(msg);
 475         return ret;
 476 }
 477
 478 struct ila_dump_iter {
 479         struct rhashtable_iter rhiter;
 480 };
 481
 482 static int ila_nl_dump_start(struct netlink_callback *cb)
 483 {
 484         struct net *net = sock_net(cb->skb->sk);
 485         struct ila_net *ilan = net_generic(net, ila_net_id);
 486         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
 487
 488         return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
 489                                     GFP_KERNEL);
 490 }
 491
 492 static int ila_nl_dump_done(struct netlink_callback *cb)
 493 {
 494         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
 495
 496         rhashtable_walk_exit(&iter->rhiter);
 497
 498         return 0;
 499 }
 500
 501 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
 502 {
 503         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args;
 504         struct rhashtable_iter *rhiter = &iter->rhiter;
 505         struct ila_map *ila;
 506         int ret;
 507
 508         ret = rhashtable_walk_start(rhiter);
 509         if (ret && ret != -EAGAIN)
 510                 goto done;
 511
 512         for (;;) {
 513                 ila = rhashtable_walk_next(rhiter);
 514
 515                 if (IS_ERR(ila)) {
 516                         if (PTR_ERR(ila) == -EAGAIN)
 517                                 continue;
 518                         ret = PTR_ERR(ila);
 519                         goto done;
 520                 } else if (!ila) {
 521                         break;
 522                 }
 523
 524                 while (ila) {
 525                         ret =  ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
 526                                              cb->nlh->nlmsg_seq, NLM_F_MULTI,
 527                                              skb, ILA_CMD_GET);
 528                         if (ret)
 529                                 goto done;
 530
 531                         ila = rcu_access_pointer(ila->next);
 532                 }
 533         }
 534
 535         ret = skb->len;
 536
 537 done:
 538         rhashtable_walk_stop(rhiter);
 539         return ret;
 540 }
 541
 542 static const struct genl_ops ila_nl_ops[] = {
 543         {
 544                 .cmd = ILA_CMD_ADD,
 545                 .doit = ila_nl_cmd_add_mapping,
 546                 .policy = ila_nl_policy,
 547                 .flags = GENL_ADMIN_PERM,
 548         },
 549         {
 550                 .cmd = ILA_CMD_DEL,
 551                 .doit = ila_nl_cmd_del_mapping,
 552                 .policy = ila_nl_policy,
 553                 .flags = GENL_ADMIN_PERM,
 554         },
 555         {
 556                 .cmd = ILA_CMD_GET,
 557                 .doit = ila_nl_cmd_get_mapping,
 558                 .start = ila_nl_dump_start,
 559                 .dumpit = ila_nl_dump,
 560                 .done = ila_nl_dump_done,
 561                 .policy = ila_nl_policy,
 562         },
 563 };
 564
 565 #define ILA_HASH_TABLE_SIZE 1024
 566
 567 static __net_init int ila_init_net(struct net *net)
 568 {
 569         int err;
 570         struct ila_net *ilan = net_generic(net, ila_net_id);
 571
 572         err = alloc_ila_locks(ilan);
 573         if (err)
 574                 return err;
 575
 576         rhashtable_init(&ilan->rhash_table, &rht_params);
 577
 578         return 0;
 579 }
 580
 581 static __net_exit void ila_exit_net(struct net *net)
 582 {
 583         struct ila_net *ilan = net_generic(net, ila_net_id);
 584
 585         rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
 586
 587         kvfree(ilan->locks);
 588
 589         if (ilan->hooks_registered)
 590                 nf_unregister_net_hooks(net, ila_nf_hook_ops,
 591                                         ARRAY_SIZE(ila_nf_hook_ops));
 592 }
 593
 594 static struct pernet_operations ila_net_ops = {
 595         .init = ila_init_net,
 596         .exit = ila_exit_net,
 597         .id   = &ila_net_id,
 598         .size = sizeof(struct ila_net),
 599 };
 600
 601 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral)
 602 {
 603         struct ila_map *ila;
 604         struct ipv6hdr *ip6h = ipv6_hdr(skb);
 605         struct net *net = dev_net(skb->dev);
 606         struct ila_net *ilan = net_generic(net, ila_net_id);
 607         struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);
 608
 609         /* Assumes skb contains a valid IPv6 header that is pulled */
 610
 611         if (!ila_addr_is_ila(iaddr)) {
 612                 /* Type indicates this is not an ILA address */
 613                 return 0;
 614         }
 615
 616         rcu_read_lock();
 617
 618         ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
 619         if (ila)
 620                 ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral);
 621
 622         rcu_read_unlock();
 623
 624         return 0;
 625 }
 626
 627 int ila_xlat_init(void)
 628 {
 629         int ret;
 630
 631         ret = register_pernet_device(&ila_net_ops);
 632         if (ret)
 633                 goto exit;
 634
 635         ret = genl_register_family_with_ops(&ila_nl_family,
 636                                             ila_nl_ops);
 637         if (ret < 0)
 638                 goto unregister;
 639
 640         return 0;
 641
 642 unregister:
 643         unregister_pernet_device(&ila_net_ops);
 644 exit:
 645         return ret;
 646 }
 647
 648 void ila_xlat_fini(void)
 649 {
 650         genl_unregister_family(&ila_nl_family);
 651         unregister_pernet_device(&ila_net_ops);
 652 }