net/ipv4/inet_hashtables.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              Generic INET transport hashtables
   7  *
   8  * Authors:     Lotsa people, from code originally in tcp
   9  *
  10  *      This program is free software; you can redistribute it and/or
  11  *      modify it under the terms of the GNU General Public License
  12  *      as published by the Free Software Foundation; either version
  13  *      2 of the License, or (at your option) any later version.
  14  */
  15
  16 #include <linux/module.h>
  17 #include <linux/random.h>
  18 #include <linux/sched.h>
  19 #include <linux/slab.h>
  20 #include <linux/wait.h>
  21 #include <linux/vmalloc.h>
  22 #include <linux/bootmem.h>
  23
  24 #include <net/addrconf.h>
  25 #include <net/inet_connection_sock.h>
  26 #include <net/inet_hashtables.h>
  27 #if IS_ENABLED(CONFIG_IPV6)
  28 #include <net/inet6_hashtables.h>
  29 #endif
  30 #include <net/secure_seq.h>
  31 #include <net/ip.h>
  32 #include <net/tcp.h>
  33 #include <net/sock_reuseport.h>
  34
  35 static u32 inet_ehashfn(const struct net *net, const __be32 laddr,
  36                         const __u16 lport, const __be32 faddr,
  37                         const __be16 fport)
  38 {
  39         static u32 inet_ehash_secret __read_mostly;
  40
  41         net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret));
  42
  43         return __inet_ehashfn(laddr, lport, faddr, fport,
  44                               inet_ehash_secret + net_hash_mix(net));
  45 }
  46
  47 /* This function handles inet_sock, but also timewait and request sockets
  48  * for IPv4/IPv6.
  49  */
  50 static u32 sk_ehashfn(const struct sock *sk)
  51 {
  52 #if IS_ENABLED(CONFIG_IPV6)
  53         if (sk->sk_family == AF_INET6 &&
  54             !ipv6_addr_v4mapped(&sk->sk_v6_daddr))
  55                 return inet6_ehashfn(sock_net(sk),
  56                                      &sk->sk_v6_rcv_saddr, sk->sk_num,
  57                                      &sk->sk_v6_daddr, sk->sk_dport);
  58 #endif
  59         return inet_ehashfn(sock_net(sk),
  60                             sk->sk_rcv_saddr, sk->sk_num,
  61                             sk->sk_daddr, sk->sk_dport);
  62 }
  63
  64 /*
  65  * Allocate and initialize a new local port bind bucket.
  66  * The bindhash mutex for snum's hash chain must be held here.
  67  */
  68 struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep,
  69                                                  struct net *net,
  70                                                  struct inet_bind_hashbucket *head,
  71                                                  const unsigned short snum)
  72 {
  73         struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
  74
  75         if (tb) {
  76                 write_pnet(&tb->ib_net, net);
  77                 tb->port      = snum;
  78                 tb->fastreuse = 0;
  79                 tb->fastreuseport = 0;
  80                 INIT_HLIST_HEAD(&tb->owners);
  81                 hlist_add_head(&tb->node, &head->chain);
  82         }
  83         return tb;
  84 }
  85
  86 /*
  87  * Caller must hold hashbucket lock for this tb with local BH disabled
  88  */
  89 void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb)
  90 {
  91         if (hlist_empty(&tb->owners)) {
  92                 __hlist_del(&tb->node);
  93                 kmem_cache_free(cachep, tb);
  94         }
  95 }
  96
  97 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
  98                     const unsigned short snum)
  99 {
 100         inet_sk(sk)->inet_num = snum;
 101         sk_add_bind_node(sk, &tb->owners);
 102         inet_csk(sk)->icsk_bind_hash = tb;
 103 }
 104
 105 /*
 106  * Get rid of any references to a local port held by the given sock.
 107  */
 108 static void __inet_put_port(struct sock *sk)
 109 {
 110         struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
 111         const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,
 112                         hashinfo->bhash_size);
 113         struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
 114         struct inet_bind_bucket *tb;
 115
 116         spin_lock(&head->lock);
 117         tb = inet_csk(sk)->icsk_bind_hash;
 118         __sk_del_bind_node(sk);
 119         inet_csk(sk)->icsk_bind_hash = NULL;
 120         inet_sk(sk)->inet_num = 0;
 121         inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
 122         spin_unlock(&head->lock);
 123 }
 124
 125 void inet_put_port(struct sock *sk)
 126 {
 127         local_bh_disable();
 128         __inet_put_port(sk);
 129         local_bh_enable();
 130 }
 131 EXPORT_SYMBOL(inet_put_port);
 132
 133 int __inet_inherit_port(const struct sock *sk, struct sock *child)
 134 {
 135         struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
 136         unsigned short port = inet_sk(child)->inet_num;
 137         const int bhash = inet_bhashfn(sock_net(sk), port,
 138                         table->bhash_size);
 139         struct inet_bind_hashbucket *head = &table->bhash[bhash];
 140         struct inet_bind_bucket *tb;
 141
 142         spin_lock(&head->lock);
 143         tb = inet_csk(sk)->icsk_bind_hash;
 144         if (unlikely(!tb)) {
 145                 spin_unlock(&head->lock);
 146                 return -ENOENT;
 147         }
 148         if (tb->port != port) {
 149                 /* NOTE: using tproxy and redirecting skbs to a proxy
 150                  * on a different listener port breaks the assumption
 151                  * that the listener socket's icsk_bind_hash is the same
 152                  * as that of the child socket. We have to look up or
 153                  * create a new bind bucket for the child here. */
 154                 inet_bind_bucket_for_each(tb, &head->chain) {
 155                         if (net_eq(ib_net(tb), sock_net(sk)) &&
 156                             tb->port == port)
 157                                 break;
 158                 }
 159                 if (!tb) {
 160                         tb = inet_bind_bucket_create(table->bind_bucket_cachep,
 161                                                      sock_net(sk), head, port);
 162                         if (!tb) {
 163                                 spin_unlock(&head->lock);
 164                                 return -ENOMEM;
 165                         }
 166                 }
 167                 inet_csk_update_fastreuse(tb, child);
 168         }
 169         inet_bind_hash(child, tb, port);
 170         spin_unlock(&head->lock);
 171
 172         return 0;
 173 }
 174 EXPORT_SYMBOL_GPL(__inet_inherit_port);
 175
 176 static struct inet_listen_hashbucket *
 177 inet_lhash2_bucket_sk(struct inet_hashinfo *h, struct sock *sk)
 178 {
 179         u32 hash;
 180
 181 #if IS_ENABLED(CONFIG_IPV6)
 182         if (sk->sk_family == AF_INET6)
 183                 hash = ipv6_portaddr_hash(sock_net(sk),
 184                                           &sk->sk_v6_rcv_saddr,
 185                                           inet_sk(sk)->inet_num);
 186         else
 187 #endif
 188                 hash = ipv4_portaddr_hash(sock_net(sk),
 189                                           inet_sk(sk)->inet_rcv_saddr,
 190                                           inet_sk(sk)->inet_num);
 191         return inet_lhash2_bucket(h, hash);
 192 }
 193
 194 static void inet_hash2(struct inet_hashinfo *h, struct sock *sk)
 195 {
 196         struct inet_listen_hashbucket *ilb2;
 197
 198         if (!h->lhash2)
 199                 return;
 200
 201         ilb2 = inet_lhash2_bucket_sk(h, sk);
 202
 203         spin_lock(&ilb2->lock);
 204         if (sk->sk_reuseport && sk->sk_family == AF_INET6)
 205                 hlist_add_tail_rcu(&inet_csk(sk)->icsk_listen_portaddr_node,
 206                                    &ilb2->head);
 207         else
 208                 hlist_add_head_rcu(&inet_csk(sk)->icsk_listen_portaddr_node,
 209                                    &ilb2->head);
 210         ilb2->count++;
 211         spin_unlock(&ilb2->lock);
 212 }
 213
 214 static void inet_unhash2(struct inet_hashinfo *h, struct sock *sk)
 215 {
 216         struct inet_listen_hashbucket *ilb2;
 217
 218         if (!h->lhash2 ||
 219             WARN_ON_ONCE(hlist_unhashed(&inet_csk(sk)->icsk_listen_portaddr_node)))
 220                 return;
 221
 222         ilb2 = inet_lhash2_bucket_sk(h, sk);
 223
 224         spin_lock(&ilb2->lock);
 225         hlist_del_init_rcu(&inet_csk(sk)->icsk_listen_portaddr_node);
 226         ilb2->count--;
 227         spin_unlock(&ilb2->lock);
 228 }
 229
 230 static inline int compute_score(struct sock *sk, struct net *net,
 231                                 const unsigned short hnum, const __be32 daddr,
 232                                 const int dif, const int sdif, bool exact_dif)
 233 {
 234         int score = -1;
 235         struct inet_sock *inet = inet_sk(sk);
 236
 237         if (net_eq(sock_net(sk), net) && inet->inet_num == hnum &&
 238                         !ipv6_only_sock(sk)) {
 239                 __be32 rcv_saddr = inet->inet_rcv_saddr;
 240                 score = sk->sk_family == PF_INET ? 2 : 1;
 241                 if (rcv_saddr) {
 242                         if (rcv_saddr != daddr)
 243                                 return -1;
 244                         score += 4;
 245                 }
 246                 if (sk->sk_bound_dev_if || exact_dif) {
 247                         bool dev_match = (sk->sk_bound_dev_if == dif ||
 248                                           sk->sk_bound_dev_if == sdif);
 249
 250                         if (!dev_match)
 251                                 return -1;
 252                         if (sk->sk_bound_dev_if)
 253                                 score += 4;
 254                 }
 255                 if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
 256                         score++;
 257         }
 258         return score;
 259 }
 260
 261 /*
 262  * Here are some nice properties to exploit here. The BSD API
 263  * does not allow a listening sock to specify the remote port nor the
 264  * remote address for the connection. So always assume those are both
 265  * wildcarded during the search since they can never be otherwise.
 266  */
 267
 268 /* called with rcu_read_lock() : No refcount taken on the socket */
 269 static struct sock *inet_lhash2_lookup(struct net *net,
 270                                 struct inet_listen_hashbucket *ilb2,
 271                                 struct sk_buff *skb, int doff,
 272                                 const __be32 saddr, __be16 sport,
 273                                 const __be32 daddr, const unsigned short hnum,
 274                                 const int dif, const int sdif)
 275 {
 276         bool exact_dif = inet_exact_dif_match(net, skb);
 277         struct inet_connection_sock *icsk;
 278         struct sock *sk, *result = NULL;
 279         int score, hiscore = 0;
 280         u32 phash = 0;
 281
 282         inet_lhash2_for_each_icsk_rcu(icsk, &ilb2->head) {
 283                 sk = (struct sock *)icsk;
 284                 score = compute_score(sk, net, hnum, daddr,
 285                                       dif, sdif, exact_dif);
 286                 if (score > hiscore) {
 287                         if (sk->sk_reuseport) {
 288                                 phash = inet_ehashfn(net, daddr, hnum,
 289                                                      saddr, sport);
 290                                 result = reuseport_select_sock(sk, phash,
 291                                                                skb, doff);
 292                                 if (result)
 293                                         return result;
 294                         }
 295                         result = sk;
 296                         hiscore = score;
 297                 }
 298         }
 299
 300         return result;
 301 }
 302
 303 struct sock *__inet_lookup_listener(struct net *net,
 304                                     struct inet_hashinfo *hashinfo,
 305                                     struct sk_buff *skb, int doff,
 306                                     const __be32 saddr, __be16 sport,
 307                                     const __be32 daddr, const unsigned short hnum,
 308                                     const int dif, const int sdif)
 309 {
 310         unsigned int hash = inet_lhashfn(net, hnum);
 311         struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash];
 312         bool exact_dif = inet_exact_dif_match(net, skb);
 313         struct inet_listen_hashbucket *ilb2;
 314         struct sock *sk, *result = NULL;
 315         struct hlist_nulls_node *node;
 316         int score, hiscore = 0;
 317         unsigned int hash2;
 318         u32 phash = 0;
 319
 320         if (ilb->count <= 10 || !hashinfo->lhash2)
 321                 goto port_lookup;
 322
 323         /* Too many sk in the ilb bucket (which is hashed by port alone).
 324          * Try lhash2 (which is hashed by port and addr) instead.
 325          */
 326
 327         hash2 = ipv4_portaddr_hash(net, daddr, hnum);
 328         ilb2 = inet_lhash2_bucket(hashinfo, hash2);
 329         if (ilb2->count > ilb->count)
 330                 goto port_lookup;
 331
 332         result = inet_lhash2_lookup(net, ilb2, skb, doff,
 333                                     saddr, sport, daddr, hnum,
 334                                     dif, sdif);
 335         if (result)
 336                 goto done;
 337
 338         /* Lookup lhash2 with INADDR_ANY */
 339
 340         hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
 341         ilb2 = inet_lhash2_bucket(hashinfo, hash2);
 342         if (ilb2->count > ilb->count)
 343                 goto port_lookup;
 344
 345         result = inet_lhash2_lookup(net, ilb2, skb, doff,
 346                                     saddr, sport, daddr, hnum,
 347                                     dif, sdif);
 348         goto done;
 349
 350 port_lookup:
 351         sk_nulls_for_each_rcu(sk, node, &ilb->nulls_head) {
 352                 score = compute_score(sk, net, hnum, daddr,
 353                                       dif, sdif, exact_dif);
 354                 if (score > hiscore) {
 355                         if (sk->sk_reuseport) {
 356                                 phash = inet_ehashfn(net, daddr, hnum,
 357                                                      saddr, sport);
 358                                 result = reuseport_select_sock(sk, phash,
 359                                                                skb, doff);
 360                                 if (result)
 361                                         goto done;
 362                         }
 363                         result = sk;
 364                         hiscore = score;
 365                 }
 366         }
 367 done:
 368         if (unlikely(IS_ERR(result)))
 369                 return NULL;
 370         return result;
 371 }
 372 EXPORT_SYMBOL_GPL(__inet_lookup_listener);
 373
 374 /* All sockets share common refcount, but have different destructors */
 375 void sock_gen_put(struct sock *sk)
 376 {
 377         if (!refcount_dec_and_test(&sk->sk_refcnt))
 378                 return;
 379
 380         if (sk->sk_state == TCP_TIME_WAIT)
 381                 inet_twsk_free(inet_twsk(sk));
 382         else if (sk->sk_state == TCP_NEW_SYN_RECV)
 383                 reqsk_free(inet_reqsk(sk));
 384         else
 385                 sk_free(sk);
 386 }
 387 EXPORT_SYMBOL_GPL(sock_gen_put);
 388
 389 void sock_edemux(struct sk_buff *skb)
 390 {
 391         sock_gen_put(skb->sk);
 392 }
 393 EXPORT_SYMBOL(sock_edemux);
 394
 395 struct sock *__inet_lookup_established(struct net *net,
 396                                   struct inet_hashinfo *hashinfo,
 397                                   const __be32 saddr, const __be16 sport,
 398                                   const __be32 daddr, const u16 hnum,
 399                                   const int dif, const int sdif)
 400 {
 401         INET_ADDR_COOKIE(acookie, saddr, daddr);
 402         const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
 403         struct sock *sk;
 404         const struct hlist_nulls_node *node;
 405         /* Optimize here for direct hit, only listening connections can
 406          * have wildcards anyways.
 407          */
 408         unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport);
 409         unsigned int slot = hash & hashinfo->ehash_mask;
 410         struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
 411
 412 begin:
 413         sk_nulls_for_each_rcu(sk, node, &head->chain) {
 414                 if (sk->sk_hash != hash)
 415                         continue;
 416                 if (likely(INET_MATCH(sk, net, acookie,
 417                                       saddr, daddr, ports, dif, sdif))) {
 418                         if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
 419                                 goto out;
 420                         if (unlikely(!INET_MATCH(sk, net, acookie,
 421                                                  saddr, daddr, ports,
 422                                                  dif, sdif))) {
 423                                 sock_gen_put(sk);
 424                                 goto begin;
 425                         }
 426                         goto found;
 427                 }
 428         }
 429         /*
 430          * if the nulls value we got at the end of this lookup is
 431          * not the expected one, we must restart lookup.
 432          * We probably met an item that was moved to another chain.
 433          */
 434         if (get_nulls_value(node) != slot)
 435                 goto begin;
 436 out:
 437         sk = NULL;
 438 found:
 439         return sk;
 440 }
 441 EXPORT_SYMBOL_GPL(__inet_lookup_established);
 442
 443 /* called with local bh disabled */
 444 static int __inet_check_established(struct inet_timewait_death_row *death_row,
 445                                     struct sock *sk, __u16 lport,
 446                                     struct inet_timewait_sock **twp)
 447 {
 448         struct inet_hashinfo *hinfo = death_row->hashinfo;
 449         struct inet_sock *inet = inet_sk(sk);
 450         __be32 daddr = inet->inet_rcv_saddr;
 451         __be32 saddr = inet->inet_daddr;
 452         int dif = sk->sk_bound_dev_if;
 453         struct net *net = sock_net(sk);
 454         int sdif = l3mdev_master_ifindex_by_index(net, dif);
 455         INET_ADDR_COOKIE(acookie, saddr, daddr);
 456         const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
 457         unsigned int hash = inet_ehashfn(net, daddr, lport,
 458                                          saddr, inet->inet_dport);
 459         struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
 460         spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
 461         struct sock *sk2;
 462         const struct hlist_nulls_node *node;
 463         struct inet_timewait_sock *tw = NULL;
 464
 465         spin_lock(lock);
 466
 467         sk_nulls_for_each(sk2, node, &head->chain) {
 468                 if (sk2->sk_hash != hash)
 469                         continue;
 470
 471                 if (likely(INET_MATCH(sk2, net, acookie,
 472                                          saddr, daddr, ports, dif, sdif))) {
 473                         if (sk2->sk_state == TCP_TIME_WAIT) {
 474                                 tw = inet_twsk(sk2);
 475                                 if (twsk_unique(sk, sk2, twp))
 476                                         break;
 477                         }
 478                         goto not_unique;
 479                 }
 480         }
 481
 482         /* Must record num and sport now. Otherwise we will see
 483          * in hash table socket with a funny identity.
 484          */
 485         inet->inet_num = lport;
 486         inet->inet_sport = htons(lport);
 487         sk->sk_hash = hash;
 488         WARN_ON(!sk_unhashed(sk));
 489         __sk_nulls_add_node_rcu(sk, &head->chain);
 490         if (tw) {
 491                 sk_nulls_del_node_init_rcu((struct sock *)tw);
 492                 __NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED);
 493         }
 494         spin_unlock(lock);
 495         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
 496
 497         if (twp) {
 498                 *twp = tw;
 499         } else if (tw) {
 500                 /* Silly. Should hash-dance instead... */
 501                 inet_twsk_deschedule_put(tw);
 502         }
 503         return 0;
 504
 505 not_unique:
 506         spin_unlock(lock);
 507         return -EADDRNOTAVAIL;
 508 }
 509
 510 static u64 inet_sk_port_offset(const struct sock *sk)
 511 {
 512         const struct inet_sock *inet = inet_sk(sk);
 513
 514         return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr,
 515                                           inet->inet_daddr,
 516                                           inet->inet_dport);
 517 }
 518
 519 /* Searches for an exsiting socket in the ehash bucket list.
 520  * Returns true if found, false otherwise.
 521  */
 522 static bool inet_ehash_lookup_by_sk(struct sock *sk,
 523                                     struct hlist_nulls_head *list)
 524 {
 525         const __portpair ports = INET_COMBINED_PORTS(sk->sk_dport, sk->sk_num);
 526         const int sdif = sk->sk_bound_dev_if;
 527         const int dif = sk->sk_bound_dev_if;
 528         const struct hlist_nulls_node *node;
 529         struct net *net = sock_net(sk);
 530         struct sock *esk;
 531
 532         INET_ADDR_COOKIE(acookie, sk->sk_daddr, sk->sk_rcv_saddr);
 533
 534         sk_nulls_for_each_rcu(esk, node, list) {
 535                 if (esk->sk_hash != sk->sk_hash)
 536                         continue;
 537                 if (sk->sk_family == AF_INET) {
 538                         if (unlikely(INET_MATCH(esk, net, acookie,
 539                                                 sk->sk_daddr,
 540                                                 sk->sk_rcv_saddr,
 541                                                 ports, dif, sdif))) {
 542                                 return true;
 543                         }
 544                 }
 545 #if IS_ENABLED(CONFIG_IPV6)
 546                 else if (sk->sk_family == AF_INET6) {
 547                         if (unlikely(INET6_MATCH(esk, net,
 548                                                  &sk->sk_v6_daddr,
 549                                                  &sk->sk_v6_rcv_saddr,
 550                                                  ports, dif, sdif))) {
 551                                 return true;
 552                         }
 553                 }
 554 #endif
 555         }
 556         return false;
 557 }
 558
 559 /* Insert a socket into ehash, and eventually remove another one
 560  * (The another one can be a SYN_RECV or TIMEWAIT)
 561  * If an existing socket already exists, socket sk is not inserted,
 562  * and sets found_dup_sk parameter to true.
 563  */
 564 bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk)
 565 {
 566         struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
 567         struct hlist_nulls_head *list;
 568         struct inet_ehash_bucket *head;
 569         spinlock_t *lock;
 570         bool ret = true;
 571
 572         WARN_ON_ONCE(!sk_unhashed(sk));
 573
 574         sk->sk_hash = sk_ehashfn(sk);
 575         head = inet_ehash_bucket(hashinfo, sk->sk_hash);
 576         list = &head->chain;
 577         lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
 578
 579         spin_lock(lock);
 580         if (osk) {
 581                 WARN_ON_ONCE(sk->sk_hash != osk->sk_hash);
 582                 ret = sk_nulls_del_node_init_rcu(osk);
 583         } else if (found_dup_sk) {
 584                 *found_dup_sk = inet_ehash_lookup_by_sk(sk, list);
 585                 if (*found_dup_sk)
 586                         ret = false;
 587         }
 588
 589         if (ret)
 590                 __sk_nulls_add_node_rcu(sk, list);
 591
 592         spin_unlock(lock);
 593
 594         return ret;
 595 }
 596
 597 bool inet_ehash_nolisten(struct sock *sk, struct sock *osk, bool *found_dup_sk)
 598 {
 599         bool ok = inet_ehash_insert(sk, osk, found_dup_sk);
 600
 601         if (ok) {
 602                 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
 603         } else {
 604                 percpu_counter_inc(sk->sk_prot->orphan_count);
 605                 inet_sk_set_state(sk, TCP_CLOSE);
 606                 sock_set_flag(sk, SOCK_DEAD);
 607                 inet_csk_destroy_sock(sk);
 608         }
 609         return ok;
 610 }
 611 EXPORT_SYMBOL_GPL(inet_ehash_nolisten);
 612
 613 static int inet_reuseport_add_sock(struct sock *sk,
 614                                    struct inet_listen_hashbucket *ilb)
 615 {
 616         struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash;
 617         const struct hlist_nulls_node *node;
 618         struct sock *sk2;
 619         kuid_t uid = sock_i_uid(sk);
 620
 621         sk_nulls_for_each_rcu(sk2, node, &ilb->nulls_head) {
 622                 if (sk2 != sk &&
 623                     sk2->sk_family == sk->sk_family &&
 624                     ipv6_only_sock(sk2) == ipv6_only_sock(sk) &&
 625                     sk2->sk_bound_dev_if == sk->sk_bound_dev_if &&
 626                     inet_csk(sk2)->icsk_bind_hash == tb &&
 627                     sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) &&
 628                     inet_rcv_saddr_equal(sk, sk2, false))
 629                         return reuseport_add_sock(sk, sk2,
 630                                                   inet_rcv_saddr_any(sk));
 631         }
 632
 633         return reuseport_alloc(sk, inet_rcv_saddr_any(sk));
 634 }
 635
 636 int __inet_hash(struct sock *sk, struct sock *osk)
 637 {
 638         struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
 639         struct inet_listen_hashbucket *ilb;
 640         int err = 0;
 641
 642         if (sk->sk_state != TCP_LISTEN) {
 643                 inet_ehash_nolisten(sk, osk, NULL);
 644                 return 0;
 645         }
 646         WARN_ON(!sk_unhashed(sk));
 647         ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
 648
 649         spin_lock(&ilb->lock);
 650         if (sk->sk_reuseport) {
 651                 err = inet_reuseport_add_sock(sk, ilb);
 652                 if (err)
 653                         goto unlock;
 654         }
 655         if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
 656                 sk->sk_family == AF_INET6)
 657                 __sk_nulls_add_node_tail_rcu(sk, &ilb->nulls_head);
 658         else
 659                 __sk_nulls_add_node_rcu(sk, &ilb->nulls_head);
 660         inet_hash2(hashinfo, sk);
 661         ilb->count++;
 662         sock_set_flag(sk, SOCK_RCU_FREE);
 663         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
 664 unlock:
 665         spin_unlock(&ilb->lock);
 666
 667         return err;
 668 }
 669 EXPORT_SYMBOL(__inet_hash);
 670
 671 int inet_hash(struct sock *sk)
 672 {
 673         int err = 0;
 674
 675         if (sk->sk_state != TCP_CLOSE) {
 676                 local_bh_disable();
 677                 err = __inet_hash(sk, NULL);
 678                 local_bh_enable();
 679         }
 680
 681         return err;
 682 }
 683 EXPORT_SYMBOL_GPL(inet_hash);
 684
 685 void inet_unhash(struct sock *sk)
 686 {
 687         struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
 688         struct inet_listen_hashbucket *ilb = NULL;
 689         spinlock_t *lock;
 690
 691         if (sk_unhashed(sk))
 692                 return;
 693
 694         if (sk->sk_state == TCP_LISTEN) {
 695                 ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
 696                 lock = &ilb->lock;
 697         } else {
 698                 lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
 699         }
 700         spin_lock_bh(lock);
 701         if (sk_unhashed(sk))
 702                 goto unlock;
 703
 704         if (rcu_access_pointer(sk->sk_reuseport_cb))
 705                 reuseport_detach_sock(sk);
 706         if (ilb) {
 707                 inet_unhash2(hashinfo, sk);
 708                 ilb->count--;
 709         }
 710         __sk_nulls_del_node_init_rcu(sk);
 711         sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
 712 unlock:
 713         spin_unlock_bh(lock);
 714 }
 715 EXPORT_SYMBOL_GPL(inet_unhash);
 716
 717 /* RFC 6056 3.3.4.  Algorithm 4: Double-Hash Port Selection Algorithm
 718  * Note that we use 32bit integers (vs RFC 'short integers')
 719  * because 2^16 is not a multiple of num_ephemeral and this
 720  * property might be used by clever attacker.
 721  * RFC claims using TABLE_LENGTH=10 buckets gives an improvement, though
 722  * attacks were since demonstrated, thus we use 65536 instead to really
 723  * give more isolation and privacy, at the expense of 256kB of kernel
 724  * memory.
 725  */
 726 #define INET_TABLE_PERTURB_SHIFT 16
 727 #define INET_TABLE_PERTURB_SIZE (1 << INET_TABLE_PERTURB_SHIFT)
 728 static u32 *table_perturb;
 729
 730 int __inet_hash_connect(struct inet_timewait_death_row *death_row,
 731                 struct sock *sk, u64 port_offset,
 732                 int (*check_established)(struct inet_timewait_death_row *,
 733                         struct sock *, __u16, struct inet_timewait_sock **))
 734 {
 735         struct inet_hashinfo *hinfo = death_row->hashinfo;
 736         struct inet_timewait_sock *tw = NULL;
 737         struct inet_bind_hashbucket *head;
 738         int port = inet_sk(sk)->inet_num;
 739         struct net *net = sock_net(sk);
 740         struct inet_bind_bucket *tb;
 741         u32 remaining, offset;
 742         int ret, i, low, high;
 743         u32 index;
 744
 745         if (port) {
 746                 head = &hinfo->bhash[inet_bhashfn(net, port,
 747                                                   hinfo->bhash_size)];
 748                 tb = inet_csk(sk)->icsk_bind_hash;
 749                 spin_lock_bh(&head->lock);
 750                 if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) {
 751                         inet_ehash_nolisten(sk, NULL, NULL);
 752                         spin_unlock_bh(&head->lock);
 753                         return 0;
 754                 }
 755                 spin_unlock(&head->lock);
 756                 /* No definite answer... Walk to established hash table */
 757                 ret = check_established(death_row, sk, port, NULL);
 758                 local_bh_enable();
 759                 return ret;
 760         }
 761
 762         inet_get_local_port_range(net, &low, &high);
 763         high++; /* [32768, 60999] -> [32768, 61000[ */
 764         remaining = high - low;
 765         if (likely(remaining > 1))
 766                 remaining &= ~1U;
 767
 768         get_random_slow_once(table_perturb,
 769                              INET_TABLE_PERTURB_SIZE * sizeof(*table_perturb));
 770         index = port_offset & (INET_TABLE_PERTURB_SIZE - 1);
 771
 772         offset = READ_ONCE(table_perturb[index]) + (port_offset >> 32);
 773         offset %= remaining;
 774
 775         /* In first pass we try ports of @low parity.
 776          * inet_csk_get_port() does the opposite choice.
 777          */
 778         offset &= ~1U;
 779 other_parity_scan:
 780         port = low + offset;
 781         for (i = 0; i < remaining; i += 2, port += 2) {
 782                 if (unlikely(port >= high))
 783                         port -= remaining;
 784                 if (inet_is_local_reserved_port(net, port))
 785                         continue;
 786                 head = &hinfo->bhash[inet_bhashfn(net, port,
 787                                                   hinfo->bhash_size)];
 788                 spin_lock_bh(&head->lock);
 789
 790                 /* Does not bother with rcv_saddr checks, because
 791                  * the established check is already unique enough.
 792                  */
 793                 inet_bind_bucket_for_each(tb, &head->chain) {
 794                         if (net_eq(ib_net(tb), net) && tb->port == port) {
 795                                 if (tb->fastreuse >= 0 ||
 796                                     tb->fastreuseport >= 0)
 797                                         goto next_port;
 798                                 WARN_ON(hlist_empty(&tb->owners));
 799                                 if (!check_established(death_row, sk,
 800                                                        port, &tw))
 801                                         goto ok;
 802                                 goto next_port;
 803                         }
 804                 }
 805
 806                 tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
 807                                              net, head, port);
 808                 if (!tb) {
 809                         spin_unlock_bh(&head->lock);
 810                         return -ENOMEM;
 811                 }
 812                 tb->fastreuse = -1;
 813                 tb->fastreuseport = -1;
 814                 goto ok;
 815 next_port:
 816                 spin_unlock_bh(&head->lock);
 817                 cond_resched();
 818         }
 819
 820         offset++;
 821         if ((offset & 1) && remaining > 1)
 822                 goto other_parity_scan;
 823
 824         return -EADDRNOTAVAIL;
 825
 826 ok:
 827         /* Here we want to add a little bit of randomness to the next source
 828          * port that will be chosen. We use a max() with a random here so that
 829          * on low contention the randomness is maximal and on high contention
 830          * it may be inexistent.
 831          */
 832         i = max_t(int, i, (prandom_u32() & 7) * 2);
 833         WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);
 834
 835         /* Head lock still held and bh's disabled */
 836         inet_bind_hash(sk, tb, port);
 837         if (sk_unhashed(sk)) {
 838                 inet_sk(sk)->inet_sport = htons(port);
 839                 inet_ehash_nolisten(sk, (struct sock *)tw, NULL);
 840         }
 841         if (tw)
 842                 inet_twsk_bind_unhash(tw, hinfo);
 843         spin_unlock(&head->lock);
 844         if (tw)
 845                 inet_twsk_deschedule_put(tw);
 846         local_bh_enable();
 847         return 0;
 848 }
 849
 850 /*
 851  * Bind a port for a connect operation and hash it.
 852  */
 853 int inet_hash_connect(struct inet_timewait_death_row *death_row,
 854                       struct sock *sk)
 855 {
 856         u64 port_offset = 0;
 857
 858         if (!inet_sk(sk)->inet_num)
 859                 port_offset = inet_sk_port_offset(sk);
 860         return __inet_hash_connect(death_row, sk, port_offset,
 861                                    __inet_check_established);
 862 }
 863 EXPORT_SYMBOL_GPL(inet_hash_connect);
 864
 865 void inet_hashinfo_init(struct inet_hashinfo *h)
 866 {
 867         int i;
 868
 869         for (i = 0; i < INET_LHTABLE_SIZE; i++) {
 870                 spin_lock_init(&h->listening_hash[i].lock);
 871                 INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].nulls_head,
 872                                       i + LISTENING_NULLS_BASE);
 873                 h->listening_hash[i].count = 0;
 874         }
 875
 876         h->lhash2 = NULL;
 877 }
 878 EXPORT_SYMBOL_GPL(inet_hashinfo_init);
 879
 880 void __init inet_hashinfo2_init(struct inet_hashinfo *h, const char *name,
 881                                 unsigned long numentries, int scale,
 882                                 unsigned long low_limit,
 883                                 unsigned long high_limit)
 884 {
 885         unsigned int i;
 886
 887         h->lhash2 = alloc_large_system_hash(name,
 888                                             sizeof(*h->lhash2),
 889                                             numentries,
 890                                             scale,
 891                                             0,
 892                                             NULL,
 893                                             &h->lhash2_mask,
 894                                             low_limit,
 895                                             high_limit);
 896
 897         for (i = 0; i <= h->lhash2_mask; i++) {
 898                 spin_lock_init(&h->lhash2[i].lock);
 899                 INIT_HLIST_HEAD(&h->lhash2[i].head);
 900                 h->lhash2[i].count = 0;
 901         }
 902
 903         /* this one is used for source ports of outgoing connections */
 904         table_perturb = kmalloc_array(INET_TABLE_PERTURB_SIZE,
 905                                       sizeof(*table_perturb), GFP_KERNEL);
 906         if (!table_perturb)
 907                 panic("TCP: failed to alloc table_perturb");
 908 }
 909
 910 int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo)
 911 {
 912         unsigned int locksz = sizeof(spinlock_t);
 913         unsigned int i, nblocks = 1;
 914
 915         if (locksz != 0) {
 916                 /* allocate 2 cache lines or at least one spinlock per cpu */
 917                 nblocks = max(2U * L1_CACHE_BYTES / locksz, 1U);
 918                 nblocks = roundup_pow_of_two(nblocks * num_possible_cpus());
 919
 920                 /* no more locks than number of hash buckets */
 921                 nblocks = min(nblocks, hashinfo->ehash_mask + 1);
 922
 923                 hashinfo->ehash_locks = kvmalloc_array(nblocks, locksz, GFP_KERNEL);
 924                 if (!hashinfo->ehash_locks)
 925                         return -ENOMEM;
 926
 927                 for (i = 0; i < nblocks; i++)
 928                         spin_lock_init(&hashinfo->ehash_locks[i]);
 929         }
 930         hashinfo->ehash_locks_mask = nblocks - 1;
 931         return 0;
 932 }
 933 EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc);