net/netfilter/nf_conntrack_core.c

   1 /* Connection state tracking for netfilter.  This is separated from,
   2    but required by, the NAT layer; it can also be used by an iptables
   3    extension. */
   4
   5 /* (C) 1999-2001 Paul `Rusty' Russell
   6  * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
   7  * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
   8  * (C) 2005-2012 Patrick McHardy <kaber@trash.net>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License version 2 as
  12  * published by the Free Software Foundation.
  13  */
  14
  15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  16
  17 #include <linux/types.h>
  18 #include <linux/netfilter.h>
  19 #include <linux/module.h>
  20 #include <linux/sched.h>
  21 #include <linux/skbuff.h>
  22 #include <linux/proc_fs.h>
  23 #include <linux/vmalloc.h>
  24 #include <linux/stddef.h>
  25 #include <linux/slab.h>
  26 #include <linux/random.h>
  27 #include <linux/jhash.h>
  28 #include <linux/siphash.h>
  29 #include <linux/err.h>
  30 #include <linux/percpu.h>
  31 #include <linux/moduleparam.h>
  32 #include <linux/notifier.h>
  33 #include <linux/kernel.h>
  34 #include <linux/netdevice.h>
  35 #include <linux/socket.h>
  36 #include <linux/mm.h>
  37 #include <linux/nsproxy.h>
  38 #include <linux/rculist_nulls.h>
  39
  40 #include <net/netfilter/nf_conntrack.h>
  41 #include <net/netfilter/nf_conntrack_l3proto.h>
  42 #include <net/netfilter/nf_conntrack_l4proto.h>
  43 #include <net/netfilter/nf_conntrack_expect.h>
  44 #include <net/netfilter/nf_conntrack_helper.h>
  45 #include <net/netfilter/nf_conntrack_seqadj.h>
  46 #include <net/netfilter/nf_conntrack_core.h>
  47 #include <net/netfilter/nf_conntrack_extend.h>
  48 #include <net/netfilter/nf_conntrack_acct.h>
  49 #include <net/netfilter/nf_conntrack_ecache.h>
  50 #include <net/netfilter/nf_conntrack_zones.h>
  51 #include <net/netfilter/nf_conntrack_timestamp.h>
  52 #include <net/netfilter/nf_conntrack_timeout.h>
  53 #include <net/netfilter/nf_conntrack_labels.h>
  54 #include <net/netfilter/nf_conntrack_synproxy.h>
  55 #include <net/netfilter/nf_nat.h>
  56 #include <net/netfilter/nf_nat_core.h>
  57 #include <net/netfilter/nf_nat_helper.h>
  58 #include <net/netns/hash.h>
  59
  60 #define NF_CONNTRACK_VERSION    "0.5.0"
  61
  62 int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
  63                                       enum nf_nat_manip_type manip,
  64                                       const struct nlattr *attr) __read_mostly;
  65 EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
  66
  67 __cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];
  68 EXPORT_SYMBOL_GPL(nf_conntrack_locks);
  69
  70 __cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);
  71 EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);
  72
  73 struct hlist_nulls_head *nf_conntrack_hash __read_mostly;
  74 EXPORT_SYMBOL_GPL(nf_conntrack_hash);
  75
  76 struct conntrack_gc_work {
  77         struct delayed_work     dwork;
  78         u32                     last_bucket;
  79         bool                    exiting;
  80         long                    next_gc_run;
  81 };
  82
  83 static __read_mostly struct kmem_cache *nf_conntrack_cachep;
  84 static __read_mostly spinlock_t nf_conntrack_locks_all_lock;
  85 static __read_mostly DEFINE_SPINLOCK(nf_conntrack_locks_all_lock);
  86 static __read_mostly bool nf_conntrack_locks_all;
  87
  88 /* every gc cycle scans at most 1/GC_MAX_BUCKETS_DIV part of table */
  89 #define GC_MAX_BUCKETS_DIV      128u
  90 /* upper bound of full table scan */
  91 #define GC_MAX_SCAN_JIFFIES     (16u * HZ)
  92 /* desired ratio of entries found to be expired */
  93 #define GC_EVICT_RATIO  50u
  94
  95 static struct conntrack_gc_work conntrack_gc_work;
  96
  97 void nf_conntrack_lock(spinlock_t *lock) __acquires(lock)
  98 {
  99         /* 1) Acquire the lock */
 100         spin_lock(lock);
 101
 102         /* 2) read nf_conntrack_locks_all, with ACQUIRE semantics
 103          * It pairs with the smp_store_release() in nf_conntrack_all_unlock()
 104          */
 105         if (likely(smp_load_acquire(&nf_conntrack_locks_all) == false))
 106                 return;
 107
 108         /* fast path failed, unlock */
 109         spin_unlock(lock);
 110
 111         /* Slow path 1) get global lock */
 112         spin_lock(&nf_conntrack_locks_all_lock);
 113
 114         /* Slow path 2) get the lock we want */
 115         spin_lock(lock);
 116
 117         /* Slow path 3) release the global lock */
 118         spin_unlock(&nf_conntrack_locks_all_lock);
 119 }
 120 EXPORT_SYMBOL_GPL(nf_conntrack_lock);
 121
 122 static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)
 123 {
 124         h1 %= CONNTRACK_LOCKS;
 125         h2 %= CONNTRACK_LOCKS;
 126         spin_unlock(&nf_conntrack_locks[h1]);
 127         if (h1 != h2)
 128                 spin_unlock(&nf_conntrack_locks[h2]);
 129 }
 130
 131 /* return true if we need to recompute hashes (in case hash table was resized) */
 132 static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,
 133                                      unsigned int h2, unsigned int sequence)
 134 {
 135         h1 %= CONNTRACK_LOCKS;
 136         h2 %= CONNTRACK_LOCKS;
 137         if (h1 <= h2) {
 138                 nf_conntrack_lock(&nf_conntrack_locks[h1]);
 139                 if (h1 != h2)
 140                         spin_lock_nested(&nf_conntrack_locks[h2],
 141                                          SINGLE_DEPTH_NESTING);
 142         } else {
 143                 nf_conntrack_lock(&nf_conntrack_locks[h2]);
 144                 spin_lock_nested(&nf_conntrack_locks[h1],
 145                                  SINGLE_DEPTH_NESTING);
 146         }
 147         if (read_seqcount_retry(&nf_conntrack_generation, sequence)) {
 148                 nf_conntrack_double_unlock(h1, h2);
 149                 return true;
 150         }
 151         return false;
 152 }
 153
 154 static void nf_conntrack_all_lock(void)
 155 {
 156         int i;
 157
 158         spin_lock(&nf_conntrack_locks_all_lock);
 159
 160         nf_conntrack_locks_all = true;
 161
 162         for (i = 0; i < CONNTRACK_LOCKS; i++) {
 163                 spin_lock(&nf_conntrack_locks[i]);
 164
 165                 /* This spin_unlock provides the "release" to ensure that
 166                  * nf_conntrack_locks_all==true is visible to everyone that
 167                  * acquired spin_lock(&nf_conntrack_locks[]).
 168                  */
 169                 spin_unlock(&nf_conntrack_locks[i]);
 170         }
 171 }
 172
 173 static void nf_conntrack_all_unlock(void)
 174 {
 175         /* All prior stores must be complete before we clear
 176          * 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock()
 177          * might observe the false value but not the entire
 178          * critical section.
 179          * It pairs with the smp_load_acquire() in nf_conntrack_lock()
 180          */
 181         smp_store_release(&nf_conntrack_locks_all, false);
 182         spin_unlock(&nf_conntrack_locks_all_lock);
 183 }
 184
 185 unsigned int nf_conntrack_htable_size __read_mostly;
 186 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
 187
 188 unsigned int nf_conntrack_max __read_mostly;
 189 seqcount_t nf_conntrack_generation __read_mostly;
 190
 191 DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
 192 EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
 193
 194 static unsigned int nf_conntrack_hash_rnd __read_mostly;
 195
 196 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple,
 197                               const struct net *net)
 198 {
 199         unsigned int n;
 200         u32 seed;
 201
 202         get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd));
 203
 204         /* The direction must be ignored, so we hash everything up to the
 205          * destination ports (which is a multiple of 4) and treat the last
 206          * three bytes manually.
 207          */
 208         seed = nf_conntrack_hash_rnd ^ net_hash_mix(net);
 209         n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
 210         return jhash2((u32 *)tuple, n, seed ^
 211                       (((__force __u16)tuple->dst.u.all << 16) |
 212                       tuple->dst.protonum));
 213 }
 214
 215 static u32 scale_hash(u32 hash)
 216 {
 217         return reciprocal_scale(hash, nf_conntrack_htable_size);
 218 }
 219
 220 static u32 __hash_conntrack(const struct net *net,
 221                             const struct nf_conntrack_tuple *tuple,
 222                             unsigned int size)
 223 {
 224         return reciprocal_scale(hash_conntrack_raw(tuple, net), size);
 225 }
 226
 227 static u32 hash_conntrack(const struct net *net,
 228                           const struct nf_conntrack_tuple *tuple)
 229 {
 230         return scale_hash(hash_conntrack_raw(tuple, net));
 231 }
 232
 233 bool
 234 nf_ct_get_tuple(const struct sk_buff *skb,
 235                 unsigned int nhoff,
 236                 unsigned int dataoff,
 237                 u_int16_t l3num,
 238                 u_int8_t protonum,
 239                 struct net *net,
 240                 struct nf_conntrack_tuple *tuple,
 241                 const struct nf_conntrack_l3proto *l3proto,
 242                 const struct nf_conntrack_l4proto *l4proto)
 243 {
 244         memset(tuple, 0, sizeof(*tuple));
 245
 246         tuple->src.l3num = l3num;
 247         if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
 248                 return false;
 249
 250         tuple->dst.protonum = protonum;
 251         tuple->dst.dir = IP_CT_DIR_ORIGINAL;
 252
 253         return l4proto->pkt_to_tuple(skb, dataoff, net, tuple);
 254 }
 255 EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
 256
 257 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
 258                        u_int16_t l3num,
 259                        struct net *net, struct nf_conntrack_tuple *tuple)
 260 {
 261         struct nf_conntrack_l3proto *l3proto;
 262         struct nf_conntrack_l4proto *l4proto;
 263         unsigned int protoff;
 264         u_int8_t protonum;
 265         int ret;
 266
 267         rcu_read_lock();
 268
 269         l3proto = __nf_ct_l3proto_find(l3num);
 270         ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
 271         if (ret != NF_ACCEPT) {
 272                 rcu_read_unlock();
 273                 return false;
 274         }
 275
 276         l4proto = __nf_ct_l4proto_find(l3num, protonum);
 277
 278         ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple,
 279                               l3proto, l4proto);
 280
 281         rcu_read_unlock();
 282         return ret;
 283 }
 284 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
 285
 286 bool
 287 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
 288                    const struct nf_conntrack_tuple *orig,
 289                    const struct nf_conntrack_l3proto *l3proto,
 290                    const struct nf_conntrack_l4proto *l4proto)
 291 {
 292         memset(inverse, 0, sizeof(*inverse));
 293
 294         inverse->src.l3num = orig->src.l3num;
 295         if (l3proto->invert_tuple(inverse, orig) == 0)
 296                 return false;
 297
 298         inverse->dst.dir = !orig->dst.dir;
 299
 300         inverse->dst.protonum = orig->dst.protonum;
 301         return l4proto->invert_tuple(inverse, orig);
 302 }
 303 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
 304
 305 /* Generate a almost-unique pseudo-id for a given conntrack.
 306  *
 307  * intentionally doesn't re-use any of the seeds used for hash
 308  * table location, we assume id gets exposed to userspace.
 309  *
 310  * Following nf_conn items do not change throughout lifetime
 311  * of the nf_conn:
 312  *
 313  * 1. nf_conn address
 314  * 2. nf_conn->master address (normally NULL)
 315  * 3. the associated net namespace
 316  * 4. the original direction tuple
 317  */
 318 u32 nf_ct_get_id(const struct nf_conn *ct)
 319 {
 320         static __read_mostly siphash_key_t ct_id_seed;
 321         unsigned long a, b, c, d;
 322
 323         net_get_random_once(&ct_id_seed, sizeof(ct_id_seed));
 324
 325         a = (unsigned long)ct;
 326         b = (unsigned long)ct->master;
 327         c = (unsigned long)nf_ct_net(ct);
 328         d = (unsigned long)siphash(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 329                                    sizeof(ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple),
 330                                    &ct_id_seed);
 331 #ifdef CONFIG_64BIT
 332         return siphash_4u64((u64)a, (u64)b, (u64)c, (u64)d, &ct_id_seed);
 333 #else
 334         return siphash_4u32((u32)a, (u32)b, (u32)c, (u32)d, &ct_id_seed);
 335 #endif
 336 }
 337 EXPORT_SYMBOL_GPL(nf_ct_get_id);
 338
 339 static void
 340 clean_from_lists(struct nf_conn *ct)
 341 {
 342         pr_debug("clean_from_lists(%p)\n", ct);
 343         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 344         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
 345
 346         /* Destroy all pending expectations */
 347         nf_ct_remove_expectations(ct);
 348 }
 349
 350 /* must be called with local_bh_disable */
 351 static void nf_ct_add_to_dying_list(struct nf_conn *ct)
 352 {
 353         struct ct_pcpu *pcpu;
 354
 355         /* add this conntrack to the (per cpu) dying list */
 356         ct->cpu = smp_processor_id();
 357         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 358
 359         spin_lock(&pcpu->lock);
 360         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 361                              &pcpu->dying);
 362         spin_unlock(&pcpu->lock);
 363 }
 364
 365 /* must be called with local_bh_disable */
 366 static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct)
 367 {
 368         struct ct_pcpu *pcpu;
 369
 370         /* add this conntrack to the (per cpu) unconfirmed list */
 371         ct->cpu = smp_processor_id();
 372         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 373
 374         spin_lock(&pcpu->lock);
 375         hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 376                              &pcpu->unconfirmed);
 377         spin_unlock(&pcpu->lock);
 378 }
 379
 380 /* must be called with local_bh_disable */
 381 static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)
 382 {
 383         struct ct_pcpu *pcpu;
 384
 385         /* We overload first tuple to link into unconfirmed or dying list.*/
 386         pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
 387
 388         spin_lock(&pcpu->lock);
 389         BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
 390         hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 391         spin_unlock(&pcpu->lock);
 392 }
 393
 394 /* Released via destroy_conntrack() */
 395 struct nf_conn *nf_ct_tmpl_alloc(struct net *net,
 396                                  const struct nf_conntrack_zone *zone,
 397                                  gfp_t flags)
 398 {
 399         struct nf_conn *tmpl;
 400
 401         tmpl = kzalloc(sizeof(*tmpl), flags);
 402         if (tmpl == NULL)
 403                 return NULL;
 404
 405         tmpl->status = IPS_TEMPLATE;
 406         write_pnet(&tmpl->ct_net, net);
 407         nf_ct_zone_add(tmpl, zone);
 408         atomic_set(&tmpl->ct_general.use, 0);
 409
 410         return tmpl;
 411 }
 412 EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc);
 413
 414 void nf_ct_tmpl_free(struct nf_conn *tmpl)
 415 {
 416         nf_ct_ext_destroy(tmpl);
 417         nf_ct_ext_free(tmpl);
 418         kfree(tmpl);
 419 }
 420 EXPORT_SYMBOL_GPL(nf_ct_tmpl_free);
 421
 422 static void
 423 destroy_conntrack(struct nf_conntrack *nfct)
 424 {
 425         struct nf_conn *ct = (struct nf_conn *)nfct;
 426         struct nf_conntrack_l4proto *l4proto;
 427
 428         pr_debug("destroy_conntrack(%p)\n", ct);
 429         NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
 430
 431         if (unlikely(nf_ct_is_template(ct))) {
 432                 nf_ct_tmpl_free(ct);
 433                 return;
 434         }
 435         rcu_read_lock();
 436         l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
 437         if (l4proto->destroy)
 438                 l4proto->destroy(ct);
 439
 440         rcu_read_unlock();
 441
 442         local_bh_disable();
 443         /* Expectations will have been removed in clean_from_lists,
 444          * except TFTP can create an expectation on the first packet,
 445          * before connection is in the list, so we need to clean here,
 446          * too.
 447          */
 448         nf_ct_remove_expectations(ct);
 449
 450         nf_ct_del_from_dying_or_unconfirmed_list(ct);
 451
 452         local_bh_enable();
 453
 454         if (ct->master)
 455                 nf_ct_put(ct->master);
 456
 457         pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
 458         nf_conntrack_free(ct);
 459 }
 460
 461 static void nf_ct_delete_from_lists(struct nf_conn *ct)
 462 {
 463         struct net *net = nf_ct_net(ct);
 464         unsigned int hash, reply_hash;
 465         unsigned int sequence;
 466
 467         nf_ct_helper_destroy(ct);
 468
 469         local_bh_disable();
 470         do {
 471                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 472                 hash = hash_conntrack(net,
 473                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 474                 reply_hash = hash_conntrack(net,
 475                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 476         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 477
 478         clean_from_lists(ct);
 479         nf_conntrack_double_unlock(hash, reply_hash);
 480
 481         nf_ct_add_to_dying_list(ct);
 482
 483         local_bh_enable();
 484 }
 485
 486 bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
 487 {
 488         struct nf_conn_tstamp *tstamp;
 489
 490         if (test_and_set_bit(IPS_DYING_BIT, &ct->status))
 491                 return false;
 492
 493         tstamp = nf_conn_tstamp_find(ct);
 494         if (tstamp) {
 495                 s32 timeout = ct->timeout - nfct_time_stamp;
 496
 497                 tstamp->stop = ktime_get_real_ns();
 498                 if (timeout < 0)
 499                         tstamp->stop -= jiffies_to_nsecs(-timeout);
 500         }
 501
 502         if (nf_conntrack_event_report(IPCT_DESTROY, ct,
 503                                     portid, report) < 0) {
 504                 /* destroy event was not delivered. nf_ct_put will
 505                  * be done by event cache worker on redelivery.
 506                  */
 507                 nf_ct_delete_from_lists(ct);
 508                 nf_conntrack_ecache_delayed_work(nf_ct_net(ct));
 509                 return false;
 510         }
 511
 512         nf_conntrack_ecache_work(nf_ct_net(ct));
 513         nf_ct_delete_from_lists(ct);
 514         nf_ct_put(ct);
 515         return true;
 516 }
 517 EXPORT_SYMBOL_GPL(nf_ct_delete);
 518
 519 static inline bool
 520 nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,
 521                 const struct nf_conntrack_tuple *tuple,
 522                 const struct nf_conntrack_zone *zone,
 523                 const struct net *net)
 524 {
 525         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 526
 527         /* A conntrack can be recreated with the equal tuple,
 528          * so we need to check that the conntrack is confirmed
 529          */
 530         return nf_ct_tuple_equal(tuple, &h->tuple) &&
 531                nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) &&
 532                nf_ct_is_confirmed(ct) &&
 533                net_eq(net, nf_ct_net(ct));
 534 }
 535
 536 /* caller must hold rcu readlock and none of the nf_conntrack_locks */
 537 static void nf_ct_gc_expired(struct nf_conn *ct)
 538 {
 539         if (!atomic_inc_not_zero(&ct->ct_general.use))
 540                 return;
 541
 542         if (nf_ct_should_gc(ct))
 543                 nf_ct_kill(ct);
 544
 545         nf_ct_put(ct);
 546 }
 547
 548 /*
 549  * Warning :
 550  * - Caller must take a reference on returned object
 551  *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
 552  */
 553 static struct nf_conntrack_tuple_hash *
 554 ____nf_conntrack_find(struct net *net, const struct nf_conntrack_zone *zone,
 555                       const struct nf_conntrack_tuple *tuple, u32 hash)
 556 {
 557         struct nf_conntrack_tuple_hash *h;
 558         struct hlist_nulls_head *ct_hash;
 559         struct hlist_nulls_node *n;
 560         unsigned int bucket, hsize;
 561
 562 begin:
 563         nf_conntrack_get_ht(&ct_hash, &hsize);
 564         bucket = reciprocal_scale(hash, hsize);
 565
 566         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) {
 567                 struct nf_conn *ct;
 568
 569                 ct = nf_ct_tuplehash_to_ctrack(h);
 570                 if (nf_ct_is_expired(ct)) {
 571                         nf_ct_gc_expired(ct);
 572                         continue;
 573                 }
 574
 575                 if (nf_ct_is_dying(ct))
 576                         continue;
 577
 578                 if (nf_ct_key_equal(h, tuple, zone, net))
 579                         return h;
 580         }
 581         /*
 582          * if the nulls value we got at the end of this lookup is
 583          * not the expected one, we must restart lookup.
 584          * We probably met an item that was moved to another chain.
 585          */
 586         if (get_nulls_value(n) != bucket) {
 587                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
 588                 goto begin;
 589         }
 590
 591         return NULL;
 592 }
 593
 594 /* Find a connection corresponding to a tuple. */
 595 static struct nf_conntrack_tuple_hash *
 596 __nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 597                         const struct nf_conntrack_tuple *tuple, u32 hash)
 598 {
 599         struct nf_conntrack_tuple_hash *h;
 600         struct nf_conn *ct;
 601
 602         rcu_read_lock();
 603 begin:
 604         h = ____nf_conntrack_find(net, zone, tuple, hash);
 605         if (h) {
 606                 ct = nf_ct_tuplehash_to_ctrack(h);
 607                 if (unlikely(nf_ct_is_dying(ct) ||
 608                              !atomic_inc_not_zero(&ct->ct_general.use)))
 609                         h = NULL;
 610                 else {
 611                         if (unlikely(!nf_ct_key_equal(h, tuple, zone, net))) {
 612                                 nf_ct_put(ct);
 613                                 goto begin;
 614                         }
 615                 }
 616         }
 617         rcu_read_unlock();
 618
 619         return h;
 620 }
 621
 622 struct nf_conntrack_tuple_hash *
 623 nf_conntrack_find_get(struct net *net, const struct nf_conntrack_zone *zone,
 624                       const struct nf_conntrack_tuple *tuple)
 625 {
 626         return __nf_conntrack_find_get(net, zone, tuple,
 627                                        hash_conntrack_raw(tuple, net));
 628 }
 629 EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
 630
 631 static void __nf_conntrack_hash_insert(struct nf_conn *ct,
 632                                        unsigned int hash,
 633                                        unsigned int reply_hash)
 634 {
 635         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 636                            &nf_conntrack_hash[hash]);
 637         hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
 638                            &nf_conntrack_hash[reply_hash]);
 639 }
 640
 641 int
 642 nf_conntrack_hash_check_insert(struct nf_conn *ct)
 643 {
 644         const struct nf_conntrack_zone *zone;
 645         struct net *net = nf_ct_net(ct);
 646         unsigned int hash, reply_hash;
 647         struct nf_conntrack_tuple_hash *h;
 648         struct hlist_nulls_node *n;
 649         unsigned int sequence;
 650
 651         zone = nf_ct_zone(ct);
 652
 653         local_bh_disable();
 654         do {
 655                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 656                 hash = hash_conntrack(net,
 657                                       &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 658                 reply_hash = hash_conntrack(net,
 659                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 660         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 661
 662         /* See if there's one in the list already, including reverse */
 663         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
 664                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 665                                     zone, net))
 666                         goto out;
 667
 668         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
 669                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 670                                     zone, net))
 671                         goto out;
 672
 673         smp_wmb();
 674         /* The caller holds a reference to this object */
 675         atomic_set(&ct->ct_general.use, 2);
 676         __nf_conntrack_hash_insert(ct, hash, reply_hash);
 677         nf_conntrack_double_unlock(hash, reply_hash);
 678         NF_CT_STAT_INC(net, insert);
 679         local_bh_enable();
 680         return 0;
 681
 682 out:
 683         nf_conntrack_double_unlock(hash, reply_hash);
 684         NF_CT_STAT_INC(net, insert_failed);
 685         local_bh_enable();
 686         return -EEXIST;
 687 }
 688 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
 689
 690 static inline void nf_ct_acct_update(struct nf_conn *ct,
 691                                      enum ip_conntrack_info ctinfo,
 692                                      unsigned int len)
 693 {
 694         struct nf_conn_acct *acct;
 695
 696         acct = nf_conn_acct_find(ct);
 697         if (acct) {
 698                 struct nf_conn_counter *counter = acct->counter;
 699
 700                 atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets);
 701                 atomic64_add(len, &counter[CTINFO2DIR(ctinfo)].bytes);
 702         }
 703 }
 704
 705 static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
 706                              const struct nf_conn *loser_ct)
 707 {
 708         struct nf_conn_acct *acct;
 709
 710         acct = nf_conn_acct_find(loser_ct);
 711         if (acct) {
 712                 struct nf_conn_counter *counter = acct->counter;
 713                 unsigned int bytes;
 714
 715                 /* u32 should be fine since we must have seen one packet. */
 716                 bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes);
 717                 nf_ct_acct_update(ct, ctinfo, bytes);
 718         }
 719 }
 720
 721 /* Resolve race on insertion if this protocol allows this. */
 722 static int nf_ct_resolve_clash(struct net *net, struct sk_buff *skb,
 723                                enum ip_conntrack_info ctinfo,
 724                                struct nf_conntrack_tuple_hash *h)
 725 {
 726         /* This is the conntrack entry already in hashes that won race. */
 727         struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 728         struct nf_conntrack_l4proto *l4proto;
 729
 730         l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
 731         if (l4proto->allow_clash &&
 732             ((ct->status & IPS_NAT_DONE_MASK) == 0) &&
 733             !nf_ct_is_dying(ct) &&
 734             atomic_inc_not_zero(&ct->ct_general.use)) {
 735                 nf_ct_acct_merge(ct, ctinfo, (struct nf_conn *)skb->nfct);
 736                 nf_conntrack_put(skb->nfct);
 737                 /* Assign conntrack already in hashes to this skbuff. Don't
 738                  * modify skb->nfctinfo to ensure consistent stateful filtering.
 739                  */
 740                 skb->nfct = &ct->ct_general;
 741                 return NF_ACCEPT;
 742         }
 743         NF_CT_STAT_INC(net, drop);
 744         return NF_DROP;
 745 }
 746
 747 /* Confirm a connection given skb; places it in hash table */
 748 int
 749 __nf_conntrack_confirm(struct sk_buff *skb)
 750 {
 751         const struct nf_conntrack_zone *zone;
 752         unsigned int hash, reply_hash;
 753         struct nf_conntrack_tuple_hash *h;
 754         struct nf_conn *ct;
 755         struct nf_conn_help *help;
 756         struct nf_conn_tstamp *tstamp;
 757         struct hlist_nulls_node *n;
 758         enum ip_conntrack_info ctinfo;
 759         struct net *net;
 760         unsigned int sequence;
 761         int ret = NF_DROP;
 762
 763         ct = nf_ct_get(skb, &ctinfo);
 764         net = nf_ct_net(ct);
 765
 766         /* ipt_REJECT uses nf_conntrack_attach to attach related
 767            ICMP/TCP RST packets in other direction.  Actual packet
 768            which created connection will be IP_CT_NEW or for an
 769            expected connection, IP_CT_RELATED. */
 770         if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
 771                 return NF_ACCEPT;
 772
 773         zone = nf_ct_zone(ct);
 774         local_bh_disable();
 775
 776         do {
 777                 sequence = read_seqcount_begin(&nf_conntrack_generation);
 778                 /* reuse the hash saved before */
 779                 hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
 780                 hash = scale_hash(hash);
 781                 reply_hash = hash_conntrack(net,
 782                                            &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 783
 784         } while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
 785
 786         /* We're not in hash table, and we refuse to set up related
 787          * connections for unconfirmed conns.  But packet copies and
 788          * REJECT will give spurious warnings here.
 789          */
 790         /* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
 791
 792         /* No external references means no one else could have
 793          * confirmed us.
 794          */
 795         NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
 796         pr_debug("Confirming conntrack %p\n", ct);
 797         /* We have to check the DYING flag after unlink to prevent
 798          * a race against nf_ct_get_next_corpse() possibly called from
 799          * user context, else we insert an already 'dead' hash, blocking
 800          * further use of that particular connection -JM.
 801          */
 802         nf_ct_del_from_dying_or_unconfirmed_list(ct);
 803
 804         if (unlikely(nf_ct_is_dying(ct))) {
 805                 nf_ct_add_to_dying_list(ct);
 806                 goto dying;
 807         }
 808
 809         /* See if there's one in the list already, including reverse:
 810            NAT could have grabbed it without realizing, since we're
 811            not in the hash.  If there is, we lost race. */
 812         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
 813                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 814                                     zone, net))
 815                         goto out;
 816
 817         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
 818                 if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 819                                     zone, net))
 820                         goto out;
 821
 822         /* Timer relative to confirmation time, not original
 823            setting time, otherwise we'd get timer wrap in
 824            weird delay cases. */
 825         ct->timeout += nfct_time_stamp;
 826         atomic_inc(&ct->ct_general.use);
 827         ct->status |= IPS_CONFIRMED;
 828
 829         /* set conntrack timestamp, if enabled. */
 830         tstamp = nf_conn_tstamp_find(ct);
 831         if (tstamp) {
 832                 if (skb->tstamp.tv64 == 0)
 833                         __net_timestamp(skb);
 834
 835                 tstamp->start = ktime_to_ns(skb->tstamp);
 836         }
 837         /* Since the lookup is lockless, hash insertion must be done after
 838          * starting the timer and setting the CONFIRMED bit. The RCU barriers
 839          * guarantee that no other CPU can find the conntrack before the above
 840          * stores are visible.
 841          */
 842         __nf_conntrack_hash_insert(ct, hash, reply_hash);
 843         nf_conntrack_double_unlock(hash, reply_hash);
 844         local_bh_enable();
 845
 846         help = nfct_help(ct);
 847         if (help && help->helper)
 848                 nf_conntrack_event_cache(IPCT_HELPER, ct);
 849
 850         nf_conntrack_event_cache(master_ct(ct) ?
 851                                  IPCT_RELATED : IPCT_NEW, ct);
 852         return NF_ACCEPT;
 853
 854 out:
 855         nf_ct_add_to_dying_list(ct);
 856         ret = nf_ct_resolve_clash(net, skb, ctinfo, h);
 857 dying:
 858         nf_conntrack_double_unlock(hash, reply_hash);
 859         NF_CT_STAT_INC(net, insert_failed);
 860         local_bh_enable();
 861         return ret;
 862 }
 863 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
 864
 865 /* Returns true if a connection correspondings to the tuple (required
 866    for NAT). */
 867 int
 868 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
 869                          const struct nf_conn *ignored_conntrack)
 870 {
 871         struct net *net = nf_ct_net(ignored_conntrack);
 872         const struct nf_conntrack_zone *zone;
 873         struct nf_conntrack_tuple_hash *h;
 874         struct hlist_nulls_head *ct_hash;
 875         unsigned int hash, hsize;
 876         struct hlist_nulls_node *n;
 877         struct nf_conn *ct;
 878
 879         zone = nf_ct_zone(ignored_conntrack);
 880
 881         rcu_read_lock();
 882  begin:
 883         nf_conntrack_get_ht(&ct_hash, &hsize);
 884         hash = __hash_conntrack(net, tuple, hsize);
 885
 886         hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) {
 887                 ct = nf_ct_tuplehash_to_ctrack(h);
 888
 889                 if (ct == ignored_conntrack)
 890                         continue;
 891
 892                 if (nf_ct_is_expired(ct)) {
 893                         nf_ct_gc_expired(ct);
 894                         continue;
 895                 }
 896
 897                 if (nf_ct_key_equal(h, tuple, zone, net)) {
 898                         /* Tuple is taken already, so caller will need to find
 899                          * a new source port to use.
 900                          *
 901                          * Only exception:
 902                          * If the *original tuples* are identical, then both
 903                          * conntracks refer to the same flow.
 904                          * This is a rare situation, it can occur e.g. when
 905                          * more than one UDP packet is sent from same socket
 906                          * in different threads.
 907                          *
 908                          * Let nf_ct_resolve_clash() deal with this later.
 909                          */
 910                         if (nf_ct_tuple_equal(&ignored_conntrack->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 911                                               &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple) &&
 912                                               nf_ct_zone_equal(ct, zone, IP_CT_DIR_ORIGINAL))
 913                                 continue;
 914
 915                         NF_CT_STAT_INC_ATOMIC(net, found);
 916                         rcu_read_unlock();
 917                         return 1;
 918                 }
 919         }
 920
 921         if (get_nulls_value(n) != hash) {
 922                 NF_CT_STAT_INC_ATOMIC(net, search_restart);
 923                 goto begin;
 924         }
 925
 926         rcu_read_unlock();
 927
 928         return 0;
 929 }
 930 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
 931
 932 #define NF_CT_EVICTION_RANGE    8
 933
 934 /* There's a small race here where we may free a just-assured
 935    connection.  Too bad: we're in trouble anyway. */
 936 static unsigned int early_drop_list(struct net *net,
 937                                     struct hlist_nulls_head *head)
 938 {
 939         struct nf_conntrack_tuple_hash *h;
 940         struct hlist_nulls_node *n;
 941         unsigned int drops = 0;
 942         struct nf_conn *tmp;
 943
 944         hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) {
 945                 tmp = nf_ct_tuplehash_to_ctrack(h);
 946
 947                 if (nf_ct_is_expired(tmp)) {
 948                         nf_ct_gc_expired(tmp);
 949                         continue;
 950                 }
 951
 952                 if (test_bit(IPS_ASSURED_BIT, &tmp->status) ||
 953                     !net_eq(nf_ct_net(tmp), net) ||
 954                     nf_ct_is_dying(tmp))
 955                         continue;
 956
 957                 if (!atomic_inc_not_zero(&tmp->ct_general.use))
 958                         continue;
 959
 960                 /* kill only if still in same netns -- might have moved due to
 961                  * SLAB_DESTROY_BY_RCU rules.
 962                  *
 963                  * We steal the timer reference.  If that fails timer has
 964                  * already fired or someone else deleted it. Just drop ref
 965                  * and move to next entry.
 966                  */
 967                 if (net_eq(nf_ct_net(tmp), net) &&
 968                     nf_ct_is_confirmed(tmp) &&
 969                     nf_ct_delete(tmp, 0, 0))
 970                         drops++;
 971
 972                 nf_ct_put(tmp);
 973         }
 974
 975         return drops;
 976 }
 977
 978 static noinline int early_drop(struct net *net, unsigned int hash)
 979 {
 980         unsigned int i, bucket;
 981
 982         for (i = 0; i < NF_CT_EVICTION_RANGE; i++) {
 983                 struct hlist_nulls_head *ct_hash;
 984                 unsigned int hsize, drops;
 985
 986                 rcu_read_lock();
 987                 nf_conntrack_get_ht(&ct_hash, &hsize);
 988                 if (!i)
 989                         bucket = reciprocal_scale(hash, hsize);
 990                 else
 991                         bucket = (bucket + 1) % hsize;
 992
 993                 drops = early_drop_list(net, &ct_hash[bucket]);
 994                 rcu_read_unlock();
 995
 996                 if (drops) {
 997                         NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops);
 998                         return true;
 999                 }
1000         }
1001
1002         return false;
1003 }
1004
1005 static void gc_worker(struct work_struct *work)
1006 {
1007         unsigned int min_interval = max(HZ / GC_MAX_BUCKETS_DIV, 1u);
1008         unsigned int i, goal, buckets = 0, expired_count = 0;
1009         struct conntrack_gc_work *gc_work;
1010         unsigned int ratio, scanned = 0;
1011         unsigned long next_run;
1012
1013         gc_work = container_of(work, struct conntrack_gc_work, dwork.work);
1014
1015         goal = nf_conntrack_htable_size / GC_MAX_BUCKETS_DIV;
1016         i = gc_work->last_bucket;
1017
1018         do {
1019                 struct nf_conntrack_tuple_hash *h;
1020                 struct hlist_nulls_head *ct_hash;
1021                 struct hlist_nulls_node *n;
1022                 unsigned int hashsz;
1023                 struct nf_conn *tmp;
1024
1025                 i++;
1026                 rcu_read_lock();
1027
1028                 nf_conntrack_get_ht(&ct_hash, &hashsz);
1029                 if (i >= hashsz)
1030                         i = 0;
1031
1032                 hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) {
1033                         tmp = nf_ct_tuplehash_to_ctrack(h);
1034
1035                         scanned++;
1036                         if (nf_ct_is_expired(tmp)) {
1037                                 nf_ct_gc_expired(tmp);
1038                                 expired_count++;
1039                                 continue;
1040                         }
1041                 }
1042
1043                 /* could check get_nulls_value() here and restart if ct
1044                  * was moved to another chain.  But given gc is best-effort
1045                  * we will just continue with next hash slot.
1046                  */
1047                 rcu_read_unlock();
1048                 cond_resched_rcu_qs();
1049         } while (++buckets < goal);
1050
1051         if (gc_work->exiting)
1052                 return;
1053
1054         /*
1055          * Eviction will normally happen from the packet path, and not
1056          * from this gc worker.
1057          *
1058          * This worker is only here to reap expired entries when system went
1059          * idle after a busy period.
1060          *
1061          * The heuristics below are supposed to balance conflicting goals:
1062          *
1063          * 1. Minimize time until we notice a stale entry
1064          * 2. Maximize scan intervals to not waste cycles
1065          *
1066          * Normally, expire ratio will be close to 0.
1067          *
1068          * As soon as a sizeable fraction of the entries have expired
1069          * increase scan frequency.
1070          */
1071         ratio = scanned ? expired_count * 100 / scanned : 0;
1072         if (ratio > GC_EVICT_RATIO) {
1073                 gc_work->next_gc_run = min_interval;
1074         } else {
1075                 unsigned int max = GC_MAX_SCAN_JIFFIES / GC_MAX_BUCKETS_DIV;
1076
1077                 BUILD_BUG_ON((GC_MAX_SCAN_JIFFIES / GC_MAX_BUCKETS_DIV) == 0);
1078
1079                 gc_work->next_gc_run += min_interval;
1080                 if (gc_work->next_gc_run > max)
1081                         gc_work->next_gc_run = max;
1082         }
1083
1084         next_run = gc_work->next_gc_run;
1085         gc_work->last_bucket = i;
1086         queue_delayed_work(system_long_wq, &gc_work->dwork, next_run);
1087 }
1088
1089 static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work)
1090 {
1091         INIT_DELAYED_WORK(&gc_work->dwork, gc_worker);
1092         gc_work->next_gc_run = HZ;
1093         gc_work->exiting = false;
1094 }
1095
1096 static struct nf_conn *
1097 __nf_conntrack_alloc(struct net *net,
1098                      const struct nf_conntrack_zone *zone,
1099                      const struct nf_conntrack_tuple *orig,
1100                      const struct nf_conntrack_tuple *repl,
1101                      gfp_t gfp, u32 hash)
1102 {
1103         struct nf_conn *ct;
1104
1105         /* We don't want any race condition at early drop stage */
1106         atomic_inc(&net->ct.count);
1107
1108         if (nf_conntrack_max &&
1109             unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
1110                 if (!early_drop(net, hash)) {
1111                         atomic_dec(&net->ct.count);
1112                         net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
1113                         return ERR_PTR(-ENOMEM);
1114                 }
1115         }
1116
1117         /*
1118          * Do not use kmem_cache_zalloc(), as this cache uses
1119          * SLAB_DESTROY_BY_RCU.
1120          */
1121         ct = kmem_cache_alloc(nf_conntrack_cachep, gfp);
1122         if (ct == NULL)
1123                 goto out;
1124
1125         spin_lock_init(&ct->lock);
1126         ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
1127         ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
1128         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
1129         /* save hash for reusing when confirming */
1130         *(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
1131         ct->status = 0;
1132         write_pnet(&ct->ct_net, net);
1133         memset(&ct->__nfct_init_offset, 0,
1134                offsetof(struct nf_conn, proto) -
1135                offsetof(struct nf_conn, __nfct_init_offset));
1136
1137         nf_ct_zone_add(ct, zone);
1138
1139         /* Because we use RCU lookups, we set ct_general.use to zero before
1140          * this is inserted in any list.
1141          */
1142         atomic_set(&ct->ct_general.use, 0);
1143         return ct;
1144 out:
1145         atomic_dec(&net->ct.count);
1146         return ERR_PTR(-ENOMEM);
1147 }
1148
1149 struct nf_conn *nf_conntrack_alloc(struct net *net,
1150                                    const struct nf_conntrack_zone *zone,
1151                                    const struct nf_conntrack_tuple *orig,
1152                                    const struct nf_conntrack_tuple *repl,
1153                                    gfp_t gfp)
1154 {
1155         return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
1156 }
1157 EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
1158
1159 void nf_conntrack_free(struct nf_conn *ct)
1160 {
1161         struct net *net = nf_ct_net(ct);
1162
1163         /* A freed object has refcnt == 0, that's
1164          * the golden rule for SLAB_DESTROY_BY_RCU
1165          */
1166         NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 0);
1167
1168         nf_ct_ext_destroy(ct);
1169         nf_ct_ext_free(ct);
1170         kmem_cache_free(nf_conntrack_cachep, ct);
1171         smp_mb__before_atomic();
1172         atomic_dec(&net->ct.count);
1173 }
1174 EXPORT_SYMBOL_GPL(nf_conntrack_free);
1175
1176
1177 /* Allocate a new conntrack: we return -ENOMEM if classification
1178    failed due to stress.  Otherwise it really is unclassifiable. */
1179 static struct nf_conntrack_tuple_hash *
1180 init_conntrack(struct net *net, struct nf_conn *tmpl,
1181                const struct nf_conntrack_tuple *tuple,
1182                struct nf_conntrack_l3proto *l3proto,
1183                struct nf_conntrack_l4proto *l4proto,
1184                struct sk_buff *skb,
1185                unsigned int dataoff, u32 hash)
1186 {
1187         struct nf_conn *ct;
1188         struct nf_conn_help *help;
1189         struct nf_conntrack_tuple repl_tuple;
1190         struct nf_conntrack_ecache *ecache;
1191         struct nf_conntrack_expect *exp = NULL;
1192         const struct nf_conntrack_zone *zone;
1193         struct nf_conn_timeout *timeout_ext;
1194         struct nf_conntrack_zone tmp;
1195         unsigned int *timeouts;
1196
1197         if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
1198                 pr_debug("Can't invert tuple.\n");
1199                 return NULL;
1200         }
1201
1202         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1203         ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
1204                                   hash);
1205         if (IS_ERR(ct))
1206                 return (struct nf_conntrack_tuple_hash *)ct;
1207
1208         if (!nf_ct_add_synproxy(ct, tmpl)) {
1209                 nf_conntrack_free(ct);
1210                 return ERR_PTR(-ENOMEM);
1211         }
1212
1213         timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
1214         if (timeout_ext) {
1215                 timeouts = nf_ct_timeout_data(timeout_ext);
1216                 if (unlikely(!timeouts))
1217                         timeouts = l4proto->get_timeouts(net);
1218         } else {
1219                 timeouts = l4proto->get_timeouts(net);
1220         }
1221
1222         if (!l4proto->new(ct, skb, dataoff, timeouts)) {
1223                 nf_conntrack_free(ct);
1224                 pr_debug("can't track with proto module\n");
1225                 return NULL;
1226         }
1227
1228         if (timeout_ext)
1229                 nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout),
1230                                       GFP_ATOMIC);
1231
1232         nf_ct_acct_ext_add(ct, GFP_ATOMIC);
1233         nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
1234         nf_ct_labels_ext_add(ct);
1235
1236         ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
1237         nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
1238                                  ecache ? ecache->expmask : 0,
1239                              GFP_ATOMIC);
1240
1241         local_bh_disable();
1242         if (net->ct.expect_count) {
1243                 spin_lock(&nf_conntrack_expect_lock);
1244                 exp = nf_ct_find_expectation(net, zone, tuple);
1245                 if (exp) {
1246                         pr_debug("expectation arrives ct=%p exp=%p\n",
1247                                  ct, exp);
1248                         /* Welcome, Mr. Bond.  We've been expecting you... */
1249                         __set_bit(IPS_EXPECTED_BIT, &ct->status);
1250                         /* exp->master safe, refcnt bumped in nf_ct_find_expectation */
1251                         ct->master = exp->master;
1252                         if (exp->helper) {
1253                                 help = nf_ct_helper_ext_add(ct, exp->helper,
1254                                                             GFP_ATOMIC);
1255                                 if (help)
1256                                         rcu_assign_pointer(help->helper, exp->helper);
1257                         }
1258
1259 #ifdef CONFIG_NF_CONNTRACK_MARK
1260                         ct->mark = exp->master->mark;
1261 #endif
1262 #ifdef CONFIG_NF_CONNTRACK_SECMARK
1263                         ct->secmark = exp->master->secmark;
1264 #endif
1265                         NF_CT_STAT_INC(net, expect_new);
1266                 }
1267                 spin_unlock(&nf_conntrack_expect_lock);
1268         }
1269         if (!exp)
1270                 __nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
1271
1272         /* Now it is inserted into the unconfirmed list, bump refcount */
1273         nf_conntrack_get(&ct->ct_general);
1274         nf_ct_add_to_unconfirmed_list(ct);
1275
1276         local_bh_enable();
1277
1278         if (exp) {
1279                 if (exp->expectfn)
1280                         exp->expectfn(ct, exp);
1281                 nf_ct_expect_put(exp);
1282         }
1283
1284         return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
1285 }
1286
1287 /* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
1288 static inline struct nf_conn *
1289 resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
1290                   struct sk_buff *skb,
1291                   unsigned int dataoff,
1292                   u_int16_t l3num,
1293                   u_int8_t protonum,
1294                   struct nf_conntrack_l3proto *l3proto,
1295                   struct nf_conntrack_l4proto *l4proto,
1296                   int *set_reply,
1297                   enum ip_conntrack_info *ctinfo)
1298 {
1299         const struct nf_conntrack_zone *zone;
1300         struct nf_conntrack_tuple tuple;
1301         struct nf_conntrack_tuple_hash *h;
1302         struct nf_conntrack_zone tmp;
1303         struct nf_conn *ct;
1304         u32 hash;
1305
1306         if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
1307                              dataoff, l3num, protonum, net, &tuple, l3proto,
1308                              l4proto)) {
1309                 pr_debug("Can't get tuple\n");
1310                 return NULL;
1311         }
1312
1313         /* look for tuple match */
1314         zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1315         hash = hash_conntrack_raw(&tuple, net);
1316         h = __nf_conntrack_find_get(net, zone, &tuple, hash);
1317         if (!h) {
1318                 h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
1319                                    skb, dataoff, hash);
1320                 if (!h)
1321                         return NULL;
1322                 if (IS_ERR(h))
1323                         return (void *)h;
1324         }
1325         ct = nf_ct_tuplehash_to_ctrack(h);
1326
1327         /* It exists; we have (non-exclusive) reference. */
1328         if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
1329                 *ctinfo = IP_CT_ESTABLISHED_REPLY;
1330                 /* Please set reply bit if this packet OK */
1331                 *set_reply = 1;
1332         } else {
1333                 /* Once we've had two way comms, always ESTABLISHED. */
1334                 if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
1335                         pr_debug("normal packet for %p\n", ct);
1336                         *ctinfo = IP_CT_ESTABLISHED;
1337                 } else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
1338                         pr_debug("related packet for %p\n", ct);
1339                         *ctinfo = IP_CT_RELATED;
1340                 } else {
1341                         pr_debug("new packet for %p\n", ct);
1342                         *ctinfo = IP_CT_NEW;
1343                 }
1344                 *set_reply = 0;
1345         }
1346         skb->nfct = &ct->ct_general;
1347         skb->nfctinfo = *ctinfo;
1348         return ct;
1349 }
1350
1351 unsigned int
1352 nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
1353                 struct sk_buff *skb)
1354 {
1355         struct nf_conn *ct, *tmpl = NULL;
1356         enum ip_conntrack_info ctinfo;
1357         struct nf_conntrack_l3proto *l3proto;
1358         struct nf_conntrack_l4proto *l4proto;
1359         unsigned int *timeouts;
1360         unsigned int dataoff;
1361         u_int8_t protonum;
1362         int set_reply = 0;
1363         int ret;
1364
1365         if (skb->nfct) {
1366                 /* Previously seen (loopback or untracked)?  Ignore. */
1367                 tmpl = (struct nf_conn *)skb->nfct;
1368                 if (!nf_ct_is_template(tmpl)) {
1369                         NF_CT_STAT_INC_ATOMIC(net, ignore);
1370                         return NF_ACCEPT;
1371                 }
1372                 skb->nfct = NULL;
1373         }
1374
1375         /* rcu_read_lock()ed by nf_hook_thresh */
1376         l3proto = __nf_ct_l3proto_find(pf);
1377         ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
1378                                    &dataoff, &protonum);
1379         if (ret <= 0) {
1380                 pr_debug("not prepared to track yet or error occurred\n");
1381                 NF_CT_STAT_INC_ATOMIC(net, error);
1382                 NF_CT_STAT_INC_ATOMIC(net, invalid);
1383                 ret = -ret;
1384                 goto out;
1385         }
1386
1387         l4proto = __nf_ct_l4proto_find(pf, protonum);
1388
1389         /* It may be an special packet, error, unclean...
1390          * inverse of the return code tells to the netfilter
1391          * core what to do with the packet. */
1392         if (l4proto->error != NULL) {
1393                 ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
1394                                      pf, hooknum);
1395                 if (ret <= 0) {
1396                         NF_CT_STAT_INC_ATOMIC(net, error);
1397                         NF_CT_STAT_INC_ATOMIC(net, invalid);
1398                         ret = -ret;
1399                         goto out;
1400                 }
1401                 /* ICMP[v6] protocol trackers may assign one conntrack. */
1402                 if (skb->nfct)
1403                         goto out;
1404         }
1405
1406         ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
1407                                l3proto, l4proto, &set_reply, &ctinfo);
1408         if (!ct) {
1409                 /* Not valid part of a connection */
1410                 NF_CT_STAT_INC_ATOMIC(net, invalid);
1411                 ret = NF_ACCEPT;
1412                 goto out;
1413         }
1414
1415         if (IS_ERR(ct)) {
1416                 /* Too stressed to deal. */
1417                 NF_CT_STAT_INC_ATOMIC(net, drop);
1418                 ret = NF_DROP;
1419                 goto out;
1420         }
1421
1422         NF_CT_ASSERT(skb->nfct);
1423
1424         /* Decide what timeout policy we want to apply to this flow. */
1425         timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
1426
1427         ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
1428         if (ret <= 0) {
1429                 /* Invalid: inverse of the return code tells
1430                  * the netfilter core what to do */
1431                 pr_debug("nf_conntrack_in: Can't track with proto module\n");
1432                 nf_conntrack_put(skb->nfct);
1433                 skb->nfct = NULL;
1434                 NF_CT_STAT_INC_ATOMIC(net, invalid);
1435                 if (ret == -NF_DROP)
1436                         NF_CT_STAT_INC_ATOMIC(net, drop);
1437                 ret = -ret;
1438                 goto out;
1439         }
1440
1441         if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1442                 nf_conntrack_event_cache(IPCT_REPLY, ct);
1443 out:
1444         if (tmpl) {
1445                 /* Special case: we have to repeat this hook, assign the
1446                  * template again to this packet. We assume that this packet
1447                  * has no conntrack assigned. This is used by nf_ct_tcp. */
1448                 if (ret == NF_REPEAT)
1449                         skb->nfct = (struct nf_conntrack *)tmpl;
1450                 else
1451                         nf_ct_put(tmpl);
1452         }
1453
1454         return ret;
1455 }
1456 EXPORT_SYMBOL_GPL(nf_conntrack_in);
1457
1458 bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
1459                           const struct nf_conntrack_tuple *orig)
1460 {
1461         bool ret;
1462
1463         rcu_read_lock();
1464         ret = nf_ct_invert_tuple(inverse, orig,
1465                                  __nf_ct_l3proto_find(orig->src.l3num),
1466                                  __nf_ct_l4proto_find(orig->src.l3num,
1467                                                       orig->dst.protonum));
1468         rcu_read_unlock();
1469         return ret;
1470 }
1471 EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
1472
1473 /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
1474    implicitly racy: see __nf_conntrack_confirm */
1475 void nf_conntrack_alter_reply(struct nf_conn *ct,
1476                               const struct nf_conntrack_tuple *newreply)
1477 {
1478         struct nf_conn_help *help = nfct_help(ct);
1479
1480         /* Should be unconfirmed, so not in hash table yet */
1481         NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
1482
1483         pr_debug("Altering reply tuple of %p to ", ct);
1484         nf_ct_dump_tuple(newreply);
1485
1486         ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1487         if (ct->master || (help && !hlist_empty(&help->expectations)))
1488                 return;
1489
1490         rcu_read_lock();
1491         __nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1492         rcu_read_unlock();
1493 }
1494 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1495
1496 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1497 void __nf_ct_refresh_acct(struct nf_conn *ct,
1498                           enum ip_conntrack_info ctinfo,
1499                           const struct sk_buff *skb,
1500                           unsigned long extra_jiffies,
1501                           int do_acct)
1502 {
1503         NF_CT_ASSERT(skb);
1504
1505         /* Only update if this is not a fixed timeout */
1506         if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1507                 goto acct;
1508
1509         /* If not in hash table, timer will not be active yet */
1510         if (nf_ct_is_confirmed(ct))
1511                 extra_jiffies += nfct_time_stamp;
1512
1513         ct->timeout = extra_jiffies;
1514 acct:
1515         if (do_acct)
1516                 nf_ct_acct_update(ct, ctinfo, skb->len);
1517 }
1518 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1519
1520 bool nf_ct_kill_acct(struct nf_conn *ct,
1521                      enum ip_conntrack_info ctinfo,
1522                      const struct sk_buff *skb)
1523 {
1524         nf_ct_acct_update(ct, ctinfo, skb->len);
1525
1526         return nf_ct_delete(ct, 0, 0);
1527 }
1528 EXPORT_SYMBOL_GPL(nf_ct_kill_acct);
1529
1530 #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1531
1532 #include <linux/netfilter/nfnetlink.h>
1533 #include <linux/netfilter/nfnetlink_conntrack.h>
1534 #include <linux/mutex.h>
1535
1536 /* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
1537  * in ip_conntrack_core, since we don't want the protocols to autoload
1538  * or depend on ctnetlink */
1539 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1540                                const struct nf_conntrack_tuple *tuple)
1541 {
1542         if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
1543             nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1544                 goto nla_put_failure;
1545         return 0;
1546
1547 nla_put_failure:
1548         return -1;
1549 }
1550 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1551
1552 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1553         [CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
1554         [CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
1555 };
1556 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1557
1558 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1559                                struct nf_conntrack_tuple *t)
1560 {
1561         if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
1562                 return -EINVAL;
1563
1564         t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1565         t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1566
1567         return 0;
1568 }
1569 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1570
1571 int nf_ct_port_nlattr_tuple_size(void)
1572 {
1573         return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1574 }
1575 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1576 #endif
1577
1578 /* Used by ipt_REJECT and ip6t_REJECT. */
1579 static void nf_conntrack_attach(struct sk_buff *nskb, const struct sk_buff *skb)
1580 {
1581         struct nf_conn *ct;
1582         enum ip_conntrack_info ctinfo;
1583
1584         /* This ICMP is in reverse direction to the packet which caused it */
1585         ct = nf_ct_get(skb, &ctinfo);
1586         if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1587                 ctinfo = IP_CT_RELATED_REPLY;
1588         else
1589                 ctinfo = IP_CT_RELATED;
1590
1591         /* Attach to new skbuff, and increment count */
1592         nskb->nfct = &ct->ct_general;
1593         nskb->nfctinfo = ctinfo;
1594         nf_conntrack_get(nskb->nfct);
1595 }
1596
1597 /* Bring out ya dead! */
1598 static struct nf_conn *
1599 get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
1600                 void *data, unsigned int *bucket)
1601 {
1602         struct nf_conntrack_tuple_hash *h;
1603         struct nf_conn *ct;
1604         struct hlist_nulls_node *n;
1605         spinlock_t *lockp;
1606
1607         for (; *bucket < nf_conntrack_htable_size; (*bucket)++) {
1608                 lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];
1609                 local_bh_disable();
1610                 nf_conntrack_lock(lockp);
1611                 if (*bucket < nf_conntrack_htable_size) {
1612                         hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnnode) {
1613                                 if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
1614                                         continue;
1615                                 ct = nf_ct_tuplehash_to_ctrack(h);
1616                                 if (net_eq(nf_ct_net(ct), net) &&
1617                                     iter(ct, data))
1618                                         goto found;
1619                         }
1620                 }
1621                 spin_unlock(lockp);
1622                 local_bh_enable();
1623                 cond_resched();
1624         }
1625
1626         return NULL;
1627 found:
1628         atomic_inc(&ct->ct_general.use);
1629         spin_unlock(lockp);
1630         local_bh_enable();
1631         return ct;
1632 }
1633
1634 static void
1635 __nf_ct_unconfirmed_destroy(struct net *net)
1636 {
1637         int cpu;
1638
1639         for_each_possible_cpu(cpu) {
1640                 struct nf_conntrack_tuple_hash *h;
1641                 struct hlist_nulls_node *n;
1642                 struct ct_pcpu *pcpu;
1643
1644                 pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
1645
1646                 spin_lock_bh(&pcpu->lock);
1647                 hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) {
1648                         struct nf_conn *ct;
1649
1650                         ct = nf_ct_tuplehash_to_ctrack(h);
1651
1652                         /* we cannot call iter() on unconfirmed list, the
1653                          * owning cpu can reallocate ct->ext at any time.
1654                          */
1655                         set_bit(IPS_DYING_BIT, &ct->status);
1656                 }
1657                 spin_unlock_bh(&pcpu->lock);
1658                 cond_resched();
1659         }
1660 }
1661
1662 void nf_ct_iterate_cleanup(struct net *net,
1663                            int (*iter)(struct nf_conn *i, void *data),
1664                            void *data, u32 portid, int report)
1665 {
1666         struct nf_conn *ct;
1667         unsigned int bucket = 0;
1668
1669         might_sleep();
1670
1671         if (atomic_read(&net->ct.count) == 0)
1672                 return;
1673
1674         __nf_ct_unconfirmed_destroy(net);
1675
1676         synchronize_net();
1677
1678         while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
1679                 /* Time to push up daises... */
1680
1681                 nf_ct_delete(ct, portid, report);
1682                 nf_ct_put(ct);
1683                 cond_resched();
1684         }
1685 }
1686 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
1687
1688 static int kill_all(struct nf_conn *i, void *data)
1689 {
1690         return 1;
1691 }
1692
1693 void nf_ct_free_hashtable(void *hash, unsigned int size)
1694 {
1695         if (is_vmalloc_addr(hash))
1696                 vfree(hash);
1697         else
1698                 free_pages((unsigned long)hash,
1699                            get_order(sizeof(struct hlist_head) * size));
1700 }
1701 EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
1702
1703 static int untrack_refs(void)
1704 {
1705         int cnt = 0, cpu;
1706
1707         for_each_possible_cpu(cpu) {
1708                 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1709
1710                 cnt += atomic_read(&ct->ct_general.use) - 1;
1711         }
1712         return cnt;
1713 }
1714
1715 void nf_conntrack_cleanup_start(void)
1716 {
1717         conntrack_gc_work.exiting = true;
1718         RCU_INIT_POINTER(ip_ct_attach, NULL);
1719 }
1720
1721 void nf_conntrack_cleanup_end(void)
1722 {
1723         RCU_INIT_POINTER(nf_ct_destroy, NULL);
1724         while (untrack_refs() > 0)
1725                 schedule();
1726
1727         cancel_delayed_work_sync(&conntrack_gc_work.dwork);
1728         nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_htable_size);
1729
1730         nf_conntrack_proto_fini();
1731         nf_conntrack_seqadj_fini();
1732         nf_conntrack_labels_fini();
1733         nf_conntrack_helper_fini();
1734         nf_conntrack_timeout_fini();
1735         nf_conntrack_ecache_fini();
1736         nf_conntrack_tstamp_fini();
1737         nf_conntrack_acct_fini();
1738         nf_conntrack_expect_fini();
1739
1740         kmem_cache_destroy(nf_conntrack_cachep);
1741 }
1742
1743 /*
1744  * Mishearing the voices in his head, our hero wonders how he's
1745  * supposed to kill the mall.
1746  */
1747 void nf_conntrack_cleanup_net(struct net *net)
1748 {
1749         LIST_HEAD(single);
1750
1751         list_add(&net->exit_list, &single);
1752         nf_conntrack_cleanup_net_list(&single);
1753 }
1754
1755 void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
1756 {
1757         int busy;
1758         struct net *net;
1759
1760         /*
1761          * This makes sure all current packets have passed through
1762          *  netfilter framework.  Roll on, two-stage module
1763          *  delete...
1764          */
1765         synchronize_net();
1766 i_see_dead_people:
1767         busy = 0;
1768         list_for_each_entry(net, net_exit_list, exit_list) {
1769                 nf_ct_iterate_cleanup(net, kill_all, NULL, 0, 0);
1770                 if (atomic_read(&net->ct.count) != 0)
1771                         busy = 1;
1772         }
1773         if (busy) {
1774                 schedule();
1775                 goto i_see_dead_people;
1776         }
1777
1778         list_for_each_entry(net, net_exit_list, exit_list) {
1779                 nf_conntrack_proto_pernet_fini(net);
1780                 nf_conntrack_helper_pernet_fini(net);
1781                 nf_conntrack_ecache_pernet_fini(net);
1782                 nf_conntrack_tstamp_pernet_fini(net);
1783                 nf_conntrack_acct_pernet_fini(net);
1784                 nf_conntrack_expect_pernet_fini(net);
1785                 free_percpu(net->ct.stat);
1786                 free_percpu(net->ct.pcpu_lists);
1787         }
1788 }
1789
1790 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
1791 {
1792         struct hlist_nulls_head *hash;
1793         unsigned int nr_slots, i;
1794         size_t sz;
1795
1796         if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head)))
1797                 return NULL;
1798
1799         BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
1800         nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
1801
1802         if (nr_slots > (UINT_MAX / sizeof(struct hlist_nulls_head)))
1803                 return NULL;
1804
1805         sz = nr_slots * sizeof(struct hlist_nulls_head);
1806         hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
1807                                         get_order(sz));
1808         if (!hash)
1809                 hash = vzalloc(sz);
1810
1811         if (hash && nulls)
1812                 for (i = 0; i < nr_slots; i++)
1813                         INIT_HLIST_NULLS_HEAD(&hash[i], i);
1814
1815         return hash;
1816 }
1817 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
1818
1819 int nf_conntrack_hash_resize(unsigned int hashsize)
1820 {
1821         int i, bucket;
1822         unsigned int old_size;
1823         struct hlist_nulls_head *hash, *old_hash;
1824         struct nf_conntrack_tuple_hash *h;
1825         struct nf_conn *ct;
1826
1827         if (!hashsize)
1828                 return -EINVAL;
1829
1830         hash = nf_ct_alloc_hashtable(&hashsize, 1);
1831         if (!hash)
1832                 return -ENOMEM;
1833
1834         old_size = nf_conntrack_htable_size;
1835         if (old_size == hashsize) {
1836                 nf_ct_free_hashtable(hash, hashsize);
1837                 return 0;
1838         }
1839
1840         local_bh_disable();
1841         nf_conntrack_all_lock();
1842         write_seqcount_begin(&nf_conntrack_generation);
1843
1844         /* Lookups in the old hash might happen in parallel, which means we
1845          * might get false negatives during connection lookup. New connections
1846          * created because of a false negative won't make it into the hash
1847          * though since that required taking the locks.
1848          */
1849
1850         for (i = 0; i < nf_conntrack_htable_size; i++) {
1851                 while (!hlist_nulls_empty(&nf_conntrack_hash[i])) {
1852                         h = hlist_nulls_entry(nf_conntrack_hash[i].first,
1853                                               struct nf_conntrack_tuple_hash, hnnode);
1854                         ct = nf_ct_tuplehash_to_ctrack(h);
1855                         hlist_nulls_del_rcu(&h->hnnode);
1856                         bucket = __hash_conntrack(nf_ct_net(ct),
1857                                                   &h->tuple, hashsize);
1858                         hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
1859                 }
1860         }
1861         old_size = nf_conntrack_htable_size;
1862         old_hash = nf_conntrack_hash;
1863
1864         nf_conntrack_hash = hash;
1865         nf_conntrack_htable_size = hashsize;
1866
1867         write_seqcount_end(&nf_conntrack_generation);
1868         nf_conntrack_all_unlock();
1869         local_bh_enable();
1870
1871         synchronize_net();
1872         nf_ct_free_hashtable(old_hash, old_size);
1873         return 0;
1874 }
1875
1876 int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
1877 {
1878         unsigned int hashsize;
1879         int rc;
1880
1881         if (current->nsproxy->net_ns != &init_net)
1882                 return -EOPNOTSUPP;
1883
1884         /* On boot, we can set this without any fancy locking. */
1885         if (!nf_conntrack_hash)
1886                 return param_set_uint(val, kp);
1887
1888         rc = kstrtouint(val, 0, &hashsize);
1889         if (rc)
1890                 return rc;
1891
1892         return nf_conntrack_hash_resize(hashsize);
1893 }
1894 EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
1895
1896 module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
1897                   &nf_conntrack_htable_size, 0600);
1898
1899 void nf_ct_untracked_status_or(unsigned long bits)
1900 {
1901         int cpu;
1902
1903         for_each_possible_cpu(cpu)
1904                 per_cpu(nf_conntrack_untracked, cpu).status |= bits;
1905 }
1906 EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
1907
1908 int nf_conntrack_init_start(void)
1909 {
1910         int max_factor = 8;
1911         int ret = -ENOMEM;
1912         int i, cpu;
1913
1914         seqcount_init(&nf_conntrack_generation);
1915
1916         for (i = 0; i < CONNTRACK_LOCKS; i++)
1917                 spin_lock_init(&nf_conntrack_locks[i]);
1918
1919         if (!nf_conntrack_htable_size) {
1920                 /* Idea from tcp.c: use 1/16384 of memory.
1921                  * On i386: 32MB machine has 512 buckets.
1922                  * >= 1GB machines have 16384 buckets.
1923                  * >= 4GB machines have 65536 buckets.
1924                  */
1925                 nf_conntrack_htable_size
1926                         = (((totalram_pages << PAGE_SHIFT) / 16384)
1927                            / sizeof(struct hlist_head));
1928                 if (totalram_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))
1929                         nf_conntrack_htable_size = 65536;
1930                 else if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
1931                         nf_conntrack_htable_size = 16384;
1932                 if (nf_conntrack_htable_size < 32)
1933                         nf_conntrack_htable_size = 32;
1934
1935                 /* Use a max. factor of four by default to get the same max as
1936                  * with the old struct list_heads. When a table size is given
1937                  * we use the old value of 8 to avoid reducing the max.
1938                  * entries. */
1939                 max_factor = 4;
1940         }
1941
1942         nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1);
1943         if (!nf_conntrack_hash)
1944                 return -ENOMEM;
1945
1946         nf_conntrack_max = max_factor * nf_conntrack_htable_size;
1947
1948         nf_conntrack_cachep = kmem_cache_create("nf_conntrack",
1949                                                 sizeof(struct nf_conn), 0,
1950                                                 SLAB_DESTROY_BY_RCU | SLAB_HWCACHE_ALIGN, NULL);
1951         if (!nf_conntrack_cachep)
1952                 goto err_cachep;
1953
1954         printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
1955                NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
1956                nf_conntrack_max);
1957
1958         ret = nf_conntrack_expect_init();
1959         if (ret < 0)
1960                 goto err_expect;
1961
1962         ret = nf_conntrack_acct_init();
1963         if (ret < 0)
1964                 goto err_acct;
1965
1966         ret = nf_conntrack_tstamp_init();
1967         if (ret < 0)
1968                 goto err_tstamp;
1969
1970         ret = nf_conntrack_ecache_init();
1971         if (ret < 0)
1972                 goto err_ecache;
1973
1974         ret = nf_conntrack_timeout_init();
1975         if (ret < 0)
1976                 goto err_timeout;
1977
1978         ret = nf_conntrack_helper_init();
1979         if (ret < 0)
1980                 goto err_helper;
1981
1982         ret = nf_conntrack_labels_init();
1983         if (ret < 0)
1984                 goto err_labels;
1985
1986         ret = nf_conntrack_seqadj_init();
1987         if (ret < 0)
1988                 goto err_seqadj;
1989
1990         ret = nf_conntrack_proto_init();
1991         if (ret < 0)
1992                 goto err_proto;
1993
1994         /* Set up fake conntrack: to never be deleted, not in any hashes */
1995         for_each_possible_cpu(cpu) {
1996                 struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1997                 write_pnet(&ct->ct_net, &init_net);
1998                 atomic_set(&ct->ct_general.use, 1);
1999         }
2000         /*  - and look it like as a confirmed connection */
2001         nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
2002
2003         conntrack_gc_work_init(&conntrack_gc_work);
2004         queue_delayed_work(system_long_wq, &conntrack_gc_work.dwork, HZ);
2005
2006         return 0;
2007
2008 err_proto:
2009         nf_conntrack_seqadj_fini();
2010 err_seqadj:
2011         nf_conntrack_labels_fini();
2012 err_labels:
2013         nf_conntrack_helper_fini();
2014 err_helper:
2015         nf_conntrack_timeout_fini();
2016 err_timeout:
2017         nf_conntrack_ecache_fini();
2018 err_ecache:
2019         nf_conntrack_tstamp_fini();
2020 err_tstamp:
2021         nf_conntrack_acct_fini();
2022 err_acct:
2023         nf_conntrack_expect_fini();
2024 err_expect:
2025         kmem_cache_destroy(nf_conntrack_cachep);
2026 err_cachep:
2027         nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_htable_size);
2028         return ret;
2029 }
2030
2031 void nf_conntrack_init_end(void)
2032 {
2033         /* For use by REJECT target */
2034         RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
2035         RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
2036 }
2037
2038 /*
2039  * We need to use special "null" values, not used in hash table
2040  */
2041 #define UNCONFIRMED_NULLS_VAL   ((1<<30)+0)
2042 #define DYING_NULLS_VAL         ((1<<30)+1)
2043 #define TEMPLATE_NULLS_VAL      ((1<<30)+2)
2044
2045 int nf_conntrack_init_net(struct net *net)
2046 {
2047         int ret = -ENOMEM;
2048         int cpu;
2049
2050         atomic_set(&net->ct.count, 0);
2051
2052         net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu);
2053         if (!net->ct.pcpu_lists)
2054                 goto err_stat;
2055
2056         for_each_possible_cpu(cpu) {
2057                 struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
2058
2059                 spin_lock_init(&pcpu->lock);
2060                 INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL);
2061                 INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL);
2062         }
2063
2064         net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
2065         if (!net->ct.stat)
2066                 goto err_pcpu_lists;
2067
2068         ret = nf_conntrack_expect_pernet_init(net);
2069         if (ret < 0)
2070                 goto err_expect;
2071         ret = nf_conntrack_acct_pernet_init(net);
2072         if (ret < 0)
2073                 goto err_acct;
2074         ret = nf_conntrack_tstamp_pernet_init(net);
2075         if (ret < 0)
2076                 goto err_tstamp;
2077         ret = nf_conntrack_ecache_pernet_init(net);
2078         if (ret < 0)
2079                 goto err_ecache;
2080         ret = nf_conntrack_helper_pernet_init(net);
2081         if (ret < 0)
2082                 goto err_helper;
2083         ret = nf_conntrack_proto_pernet_init(net);
2084         if (ret < 0)
2085                 goto err_proto;
2086         return 0;
2087
2088 err_proto:
2089         nf_conntrack_helper_pernet_fini(net);
2090 err_helper:
2091         nf_conntrack_ecache_pernet_fini(net);
2092 err_ecache:
2093         nf_conntrack_tstamp_pernet_fini(net);
2094 err_tstamp:
2095         nf_conntrack_acct_pernet_fini(net);
2096 err_acct:
2097         nf_conntrack_expect_pernet_fini(net);
2098 err_expect:
2099         free_percpu(net->ct.stat);
2100 err_pcpu_lists:
2101         free_percpu(net->ct.pcpu_lists);
2102 err_stat:
2103         return ret;
2104 }