net/ipv4/inet_connection_sock.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              Support for INET connection oriented protocols.
   7  *
   8  * Authors:     See the TCP sources
   9  *
  10  *              This program is free software; you can redistribute it and/or
  11  *              modify it under the terms of the GNU General Public License
  12  *              as published by the Free Software Foundation; either version
  13  *              2 of the License, or(at your option) any later version.
  14  */
  15
  16 #include <linux/module.h>
  17 #include <linux/jhash.h>
  18
  19 #include <net/inet_connection_sock.h>
  20 #include <net/inet_hashtables.h>
  21 #include <net/inet_timewait_sock.h>
  22 #include <net/ip.h>
  23 #include <net/route.h>
  24 #include <net/tcp_states.h>
  25 #include <net/xfrm.h>
  26 #include <net/tcp.h>
  27 #include <net/sock_reuseport.h>
  28
  29 #ifdef INET_CSK_DEBUG
  30 const char inet_csk_timer_bug_msg[] = "inet_csk BUG: unknown timer value\n";
  31 EXPORT_SYMBOL(inet_csk_timer_bug_msg);
  32 #endif
  33
  34 void inet_get_local_port_range(struct net *net, int *low, int *high)
  35 {
  36         unsigned int seq;
  37
  38         do {
  39                 seq = read_seqbegin(&net->ipv4.ip_local_ports.lock);
  40
  41                 *low = net->ipv4.ip_local_ports.range[0];
  42                 *high = net->ipv4.ip_local_ports.range[1];
  43         } while (read_seqretry(&net->ipv4.ip_local_ports.lock, seq));
  44 }
  45 EXPORT_SYMBOL(inet_get_local_port_range);
  46
  47 int inet_csk_bind_conflict(const struct sock *sk,
  48                            const struct inet_bind_bucket *tb, bool relax)
  49 {
  50         struct sock *sk2;
  51         int reuse = sk->sk_reuse;
  52         int reuseport = sk->sk_reuseport;
  53         kuid_t uid = sock_i_uid((struct sock *)sk);
  54
  55         /*
  56          * Unlike other sk lookup places we do not check
  57          * for sk_net here, since _all_ the socks listed
  58          * in tb->owners list belong to the same net - the
  59          * one this bucket belongs to.
  60          */
  61
  62         sk_for_each_bound(sk2, &tb->owners) {
  63                 if (sk != sk2 &&
  64                     !inet_v6_ipv6only(sk2) &&
  65                     (!sk->sk_bound_dev_if ||
  66                      !sk2->sk_bound_dev_if ||
  67                      sk->sk_bound_dev_if == sk2->sk_bound_dev_if)) {
  68                         if ((!reuse || !sk2->sk_reuse ||
  69                             sk2->sk_state == TCP_LISTEN) &&
  70                             (!reuseport || !sk2->sk_reuseport ||
  71                              rcu_access_pointer(sk->sk_reuseport_cb) ||
  72                              (sk2->sk_state != TCP_TIME_WAIT &&
  73                              !uid_eq(uid, sock_i_uid(sk2))))) {
  74
  75                                 if (!sk2->sk_rcv_saddr || !sk->sk_rcv_saddr ||
  76                                     sk2->sk_rcv_saddr == sk->sk_rcv_saddr)
  77                                         break;
  78                         }
  79                         if (!relax && reuse && sk2->sk_reuse &&
  80                             sk2->sk_state != TCP_LISTEN) {
  81
  82                                 if (!sk2->sk_rcv_saddr || !sk->sk_rcv_saddr ||
  83                                     sk2->sk_rcv_saddr == sk->sk_rcv_saddr)
  84                                         break;
  85                         }
  86                 }
  87         }
  88         return sk2 != NULL;
  89 }
  90 EXPORT_SYMBOL_GPL(inet_csk_bind_conflict);
  91
  92 void inet_csk_update_fastreuse(struct inet_bind_bucket *tb,
  93                                struct sock *sk)
  94 {
  95         kuid_t uid = sock_i_uid(sk);
  96         bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
  97
  98         if (!hlist_empty(&tb->owners)) {
  99                 if (!reuse)
 100                         tb->fastreuse = 0;
 101                 if (!sk->sk_reuseport || !uid_eq(tb->fastuid, uid))
 102                         tb->fastreuseport = 0;
 103         } else {
 104                 tb->fastreuse = reuse;
 105                 if (sk->sk_reuseport) {
 106                         tb->fastreuseport = 1;
 107                         tb->fastuid = uid;
 108                 } else {
 109                         tb->fastreuseport = 0;
 110                 }
 111         }
 112 }
 113
 114 /* Obtain a reference to a local port for the given sock,
 115  * if snum is zero it means select any available local port.
 116  * We try to allocate an odd port (and leave even ports for connect())
 117  */
 118 int inet_csk_get_port(struct sock *sk, unsigned short snum)
 119 {
 120         bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
 121         struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
 122         int ret = 1, attempts = 5, port = snum;
 123         int smallest_size = -1, smallest_port;
 124         struct inet_bind_hashbucket *head;
 125         struct net *net = sock_net(sk);
 126         int i, low, high, attempt_half;
 127         struct inet_bind_bucket *tb;
 128         kuid_t uid = sock_i_uid(sk);
 129         u32 remaining, offset;
 130
 131         if (port) {
 132 have_port:
 133                 head = &hinfo->bhash[inet_bhashfn(net, port,
 134                                                   hinfo->bhash_size)];
 135                 spin_lock_bh(&head->lock);
 136                 inet_bind_bucket_for_each(tb, &head->chain)
 137                         if (net_eq(ib_net(tb), net) && tb->port == port)
 138                                 goto tb_found;
 139
 140                 goto tb_not_found;
 141         }
 142 again:
 143         attempt_half = (sk->sk_reuse == SK_CAN_REUSE) ? 1 : 0;
 144 other_half_scan:
 145         inet_get_local_port_range(net, &low, &high);
 146         high++; /* [32768, 60999] -> [32768, 61000[ */
 147         if (high - low < 4)
 148                 attempt_half = 0;
 149         if (attempt_half) {
 150                 int half = low + (((high - low) >> 2) << 1);
 151
 152                 if (attempt_half == 1)
 153                         high = half;
 154                 else
 155                         low = half;
 156         }
 157         remaining = high - low;
 158         if (likely(remaining > 1))
 159                 remaining &= ~1U;
 160
 161         offset = prandom_u32() % remaining;
 162         /* __inet_hash_connect() favors ports having @low parity
 163          * We do the opposite to not pollute connect() users.
 164          */
 165         offset |= 1U;
 166         smallest_size = -1;
 167         smallest_port = low; /* avoid compiler warning */
 168
 169 other_parity_scan:
 170         port = low + offset;
 171         for (i = 0; i < remaining; i += 2, port += 2) {
 172                 if (unlikely(port >= high))
 173                         port -= remaining;
 174                 if (inet_is_local_reserved_port(net, port))
 175                         continue;
 176                 head = &hinfo->bhash[inet_bhashfn(net, port,
 177                                                   hinfo->bhash_size)];
 178                 spin_lock_bh(&head->lock);
 179                 inet_bind_bucket_for_each(tb, &head->chain)
 180                         if (net_eq(ib_net(tb), net) && tb->port == port) {
 181                                 if (((tb->fastreuse > 0 && reuse) ||
 182                                      (tb->fastreuseport > 0 &&
 183                                       sk->sk_reuseport &&
 184                                       !rcu_access_pointer(sk->sk_reuseport_cb) &&
 185                                       uid_eq(tb->fastuid, uid))) &&
 186                                     (tb->num_owners < smallest_size || smallest_size == -1)) {
 187                                         smallest_size = tb->num_owners;
 188                                         smallest_port = port;
 189                                 }
 190                                 if (!inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, false))
 191                                         goto tb_found;
 192                                 goto next_port;
 193                         }
 194                 goto tb_not_found;
 195 next_port:
 196                 spin_unlock_bh(&head->lock);
 197                 cond_resched();
 198         }
 199
 200         if (smallest_size != -1) {
 201                 port = smallest_port;
 202                 goto have_port;
 203         }
 204         offset--;
 205         if (!(offset & 1))
 206                 goto other_parity_scan;
 207
 208         if (attempt_half == 1) {
 209                 /* OK we now try the upper half of the range */
 210                 attempt_half = 2;
 211                 goto other_half_scan;
 212         }
 213         return ret;
 214
 215 tb_not_found:
 216         tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
 217                                      net, head, port);
 218         if (!tb)
 219                 goto fail_unlock;
 220 tb_found:
 221         if (!hlist_empty(&tb->owners)) {
 222                 if (sk->sk_reuse == SK_FORCE_REUSE)
 223                         goto success;
 224
 225                 if (((tb->fastreuse > 0 && reuse) ||
 226                      (tb->fastreuseport > 0 &&
 227                       !rcu_access_pointer(sk->sk_reuseport_cb) &&
 228                       sk->sk_reuseport && uid_eq(tb->fastuid, uid))) &&
 229                     smallest_size == -1)
 230                         goto success;
 231                 if (inet_csk(sk)->icsk_af_ops->bind_conflict(sk, tb, true)) {
 232                         if ((reuse ||
 233                              (tb->fastreuseport > 0 &&
 234                               sk->sk_reuseport &&
 235                               !rcu_access_pointer(sk->sk_reuseport_cb) &&
 236                               uid_eq(tb->fastuid, uid))) &&
 237                             smallest_size != -1 && --attempts >= 0) {
 238                                 spin_unlock_bh(&head->lock);
 239                                 goto again;
 240                         }
 241                         goto fail_unlock;
 242                 }
 243         }
 244
 245         inet_csk_update_fastreuse(tb, sk);
 246
 247 success:
 248         if (!inet_csk(sk)->icsk_bind_hash)
 249                 inet_bind_hash(sk, tb, port);
 250         WARN_ON(inet_csk(sk)->icsk_bind_hash != tb);
 251         ret = 0;
 252
 253 fail_unlock:
 254         spin_unlock_bh(&head->lock);
 255         return ret;
 256 }
 257 EXPORT_SYMBOL_GPL(inet_csk_get_port);
 258
 259 /*
 260  * Wait for an incoming connection, avoid race conditions. This must be called
 261  * with the socket locked.
 262  */
 263 static int inet_csk_wait_for_connect(struct sock *sk, long timeo)
 264 {
 265         struct inet_connection_sock *icsk = inet_csk(sk);
 266         DEFINE_WAIT(wait);
 267         int err;
 268
 269         /*
 270          * True wake-one mechanism for incoming connections: only
 271          * one process gets woken up, not the 'whole herd'.
 272          * Since we do not 'race & poll' for established sockets
 273          * anymore, the common case will execute the loop only once.
 274          *
 275          * Subtle issue: "add_wait_queue_exclusive()" will be added
 276          * after any current non-exclusive waiters, and we know that
 277          * it will always _stay_ after any new non-exclusive waiters
 278          * because all non-exclusive waiters are added at the
 279          * beginning of the wait-queue. As such, it's ok to "drop"
 280          * our exclusiveness temporarily when we get woken up without
 281          * having to remove and re-insert us on the wait queue.
 282          */
 283         for (;;) {
 284                 prepare_to_wait_exclusive(sk_sleep(sk), &wait,
 285                                           TASK_INTERRUPTIBLE);
 286                 release_sock(sk);
 287                 if (reqsk_queue_empty(&icsk->icsk_accept_queue))
 288                         timeo = schedule_timeout(timeo);
 289                 sched_annotate_sleep();
 290                 lock_sock(sk);
 291                 err = 0;
 292                 if (!reqsk_queue_empty(&icsk->icsk_accept_queue))
 293                         break;
 294                 err = -EINVAL;
 295                 if (sk->sk_state != TCP_LISTEN)
 296                         break;
 297                 err = sock_intr_errno(timeo);
 298                 if (signal_pending(current))
 299                         break;
 300                 err = -EAGAIN;
 301                 if (!timeo)
 302                         break;
 303         }
 304         finish_wait(sk_sleep(sk), &wait);
 305         return err;
 306 }
 307
 308 /*
 309  * This will accept the next outstanding connection.
 310  */
 311 struct sock *inet_csk_accept(struct sock *sk, int flags, int *err)
 312 {
 313         struct inet_connection_sock *icsk = inet_csk(sk);
 314         struct request_sock_queue *queue = &icsk->icsk_accept_queue;
 315         struct request_sock *req;
 316         struct sock *newsk;
 317         int error;
 318
 319         lock_sock(sk);
 320
 321         /* We need to make sure that this socket is listening,
 322          * and that it has something pending.
 323          */
 324         error = -EINVAL;
 325         if (sk->sk_state != TCP_LISTEN)
 326                 goto out_err;
 327
 328         /* Find already established connection */
 329         if (reqsk_queue_empty(queue)) {
 330                 long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
 331
 332                 /* If this is a non blocking socket don't sleep */
 333                 error = -EAGAIN;
 334                 if (!timeo)
 335                         goto out_err;
 336
 337                 error = inet_csk_wait_for_connect(sk, timeo);
 338                 if (error)
 339                         goto out_err;
 340         }
 341         req = reqsk_queue_remove(queue, sk);
 342         newsk = req->sk;
 343
 344         if (sk->sk_protocol == IPPROTO_TCP &&
 345             tcp_rsk(req)->tfo_listener) {
 346                 spin_lock_bh(&queue->fastopenq.lock);
 347                 if (tcp_rsk(req)->tfo_listener) {
 348                         /* We are still waiting for the final ACK from 3WHS
 349                          * so can't free req now. Instead, we set req->sk to
 350                          * NULL to signify that the child socket is taken
 351                          * so reqsk_fastopen_remove() will free the req
 352                          * when 3WHS finishes (or is aborted).
 353                          */
 354                         req->sk = NULL;
 355                         req = NULL;
 356                 }
 357                 spin_unlock_bh(&queue->fastopenq.lock);
 358         }
 359 out:
 360         release_sock(sk);
 361         if (req)
 362                 reqsk_put(req);
 363         return newsk;
 364 out_err:
 365         newsk = NULL;
 366         req = NULL;
 367         *err = error;
 368         goto out;
 369 }
 370 EXPORT_SYMBOL(inet_csk_accept);
 371
 372 /*
 373  * Using different timers for retransmit, delayed acks and probes
 374  * We may wish use just one timer maintaining a list of expire jiffies
 375  * to optimize.
 376  */
 377 void inet_csk_init_xmit_timers(struct sock *sk,
 378                                void (*retransmit_handler)(unsigned long),
 379                                void (*delack_handler)(unsigned long),
 380                                void (*keepalive_handler)(unsigned long))
 381 {
 382         struct inet_connection_sock *icsk = inet_csk(sk);
 383
 384         setup_timer(&icsk->icsk_retransmit_timer, retransmit_handler,
 385                         (unsigned long)sk);
 386         setup_timer(&icsk->icsk_delack_timer, delack_handler,
 387                         (unsigned long)sk);
 388         setup_timer(&sk->sk_timer, keepalive_handler, (unsigned long)sk);
 389         icsk->icsk_pending = icsk->icsk_ack.pending = 0;
 390 }
 391 EXPORT_SYMBOL(inet_csk_init_xmit_timers);
 392
 393 void inet_csk_clear_xmit_timers(struct sock *sk)
 394 {
 395         struct inet_connection_sock *icsk = inet_csk(sk);
 396
 397         icsk->icsk_pending = icsk->icsk_ack.pending = icsk->icsk_ack.blocked = 0;
 398
 399         sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
 400         sk_stop_timer(sk, &icsk->icsk_delack_timer);
 401         sk_stop_timer(sk, &sk->sk_timer);
 402 }
 403 EXPORT_SYMBOL(inet_csk_clear_xmit_timers);
 404
 405 void inet_csk_delete_keepalive_timer(struct sock *sk)
 406 {
 407         sk_stop_timer(sk, &sk->sk_timer);
 408 }
 409 EXPORT_SYMBOL(inet_csk_delete_keepalive_timer);
 410
 411 void inet_csk_reset_keepalive_timer(struct sock *sk, unsigned long len)
 412 {
 413         sk_reset_timer(sk, &sk->sk_timer, jiffies + len);
 414 }
 415 EXPORT_SYMBOL(inet_csk_reset_keepalive_timer);
 416
 417 struct dst_entry *inet_csk_route_req(const struct sock *sk,
 418                                      struct flowi4 *fl4,
 419                                      const struct request_sock *req)
 420 {
 421         const struct inet_request_sock *ireq = inet_rsk(req);
 422         struct net *net = read_pnet(&ireq->ireq_net);
 423         struct ip_options_rcu *opt;
 424         struct rtable *rt;
 425
 426         rcu_read_lock();
 427         opt = rcu_dereference(ireq->ireq_opt);
 428
 429         flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark,
 430                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 431                            sk->sk_protocol, inet_sk_flowi_flags(sk),
 432                            (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr,
 433                            ireq->ir_loc_addr, ireq->ir_rmt_port,
 434                            htons(ireq->ir_num));
 435         security_req_classify_flow(req, flowi4_to_flowi(fl4));
 436         rt = ip_route_output_flow(net, fl4, sk);
 437         if (IS_ERR(rt))
 438                 goto no_route;
 439         if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway)
 440                 goto route_err;
 441         rcu_read_unlock();
 442         return &rt->dst;
 443
 444 route_err:
 445         ip_rt_put(rt);
 446 no_route:
 447         rcu_read_unlock();
 448         __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
 449         return NULL;
 450 }
 451 EXPORT_SYMBOL_GPL(inet_csk_route_req);
 452
 453 struct dst_entry *inet_csk_route_child_sock(const struct sock *sk,
 454                                             struct sock *newsk,
 455                                             const struct request_sock *req)
 456 {
 457         const struct inet_request_sock *ireq = inet_rsk(req);
 458         struct net *net = read_pnet(&ireq->ireq_net);
 459         struct inet_sock *newinet = inet_sk(newsk);
 460         struct ip_options_rcu *opt;
 461         struct flowi4 *fl4;
 462         struct rtable *rt;
 463
 464         opt = rcu_dereference(ireq->ireq_opt);
 465         fl4 = &newinet->cork.fl.u.ip4;
 466
 467         flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark,
 468                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 469                            sk->sk_protocol, inet_sk_flowi_flags(sk),
 470                            (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr,
 471                            ireq->ir_loc_addr, ireq->ir_rmt_port,
 472                            htons(ireq->ir_num));
 473         security_req_classify_flow(req, flowi4_to_flowi(fl4));
 474         rt = ip_route_output_flow(net, fl4, sk);
 475         if (IS_ERR(rt))
 476                 goto no_route;
 477         if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway)
 478                 goto route_err;
 479         return &rt->dst;
 480
 481 route_err:
 482         ip_rt_put(rt);
 483 no_route:
 484         __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
 485         return NULL;
 486 }
 487 EXPORT_SYMBOL_GPL(inet_csk_route_child_sock);
 488
 489 #if IS_ENABLED(CONFIG_IPV6)
 490 #define AF_INET_FAMILY(fam) ((fam) == AF_INET)
 491 #else
 492 #define AF_INET_FAMILY(fam) true
 493 #endif
 494
 495 /* Decide when to expire the request and when to resend SYN-ACK */
 496 static inline void syn_ack_recalc(struct request_sock *req, const int thresh,
 497                                   const int max_retries,
 498                                   const u8 rskq_defer_accept,
 499                                   int *expire, int *resend)
 500 {
 501         if (!rskq_defer_accept) {
 502                 *expire = req->num_timeout >= thresh;
 503                 *resend = 1;
 504                 return;
 505         }
 506         *expire = req->num_timeout >= thresh &&
 507                   (!inet_rsk(req)->acked || req->num_timeout >= max_retries);
 508         /*
 509          * Do not resend while waiting for data after ACK,
 510          * start to resend on end of deferring period to give
 511          * last chance for data or ACK to create established socket.
 512          */
 513         *resend = !inet_rsk(req)->acked ||
 514                   req->num_timeout >= rskq_defer_accept - 1;
 515 }
 516
 517 int inet_rtx_syn_ack(const struct sock *parent, struct request_sock *req)
 518 {
 519         int err = req->rsk_ops->rtx_syn_ack(parent, req);
 520
 521         if (!err)
 522                 req->num_retrans++;
 523         return err;
 524 }
 525 EXPORT_SYMBOL(inet_rtx_syn_ack);
 526
 527 /* return true if req was found in the ehash table */
 528 static bool reqsk_queue_unlink(struct request_sock_queue *queue,
 529                                struct request_sock *req)
 530 {
 531         struct inet_hashinfo *hashinfo = req_to_sk(req)->sk_prot->h.hashinfo;
 532         bool found = false;
 533
 534         if (sk_hashed(req_to_sk(req))) {
 535                 spinlock_t *lock = inet_ehash_lockp(hashinfo, req->rsk_hash);
 536
 537                 spin_lock(lock);
 538                 found = __sk_nulls_del_node_init_rcu(req_to_sk(req));
 539                 spin_unlock(lock);
 540         }
 541         if (timer_pending(&req->rsk_timer) && del_timer_sync(&req->rsk_timer))
 542                 reqsk_put(req);
 543         return found;
 544 }
 545
 546 void inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req)
 547 {
 548         if (reqsk_queue_unlink(&inet_csk(sk)->icsk_accept_queue, req)) {
 549                 reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req);
 550                 reqsk_put(req);
 551         }
 552 }
 553 EXPORT_SYMBOL(inet_csk_reqsk_queue_drop);
 554
 555 void inet_csk_reqsk_queue_drop_and_put(struct sock *sk, struct request_sock *req)
 556 {
 557         inet_csk_reqsk_queue_drop(sk, req);
 558         reqsk_put(req);
 559 }
 560 EXPORT_SYMBOL(inet_csk_reqsk_queue_drop_and_put);
 561
 562 static void reqsk_timer_handler(unsigned long data)
 563 {
 564         struct request_sock *req = (struct request_sock *)data;
 565         struct sock *sk_listener = req->rsk_listener;
 566         struct net *net = sock_net(sk_listener);
 567         struct inet_connection_sock *icsk = inet_csk(sk_listener);
 568         struct request_sock_queue *queue = &icsk->icsk_accept_queue;
 569         int qlen, expire = 0, resend = 0;
 570         int max_retries, thresh;
 571         u8 defer_accept;
 572
 573         if (sk_state_load(sk_listener) != TCP_LISTEN)
 574                 goto drop;
 575
 576         max_retries = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_synack_retries;
 577         thresh = max_retries;
 578         /* Normally all the openreqs are young and become mature
 579          * (i.e. converted to established socket) for first timeout.
 580          * If synack was not acknowledged for 1 second, it means
 581          * one of the following things: synack was lost, ack was lost,
 582          * rtt is high or nobody planned to ack (i.e. synflood).
 583          * When server is a bit loaded, queue is populated with old
 584          * open requests, reducing effective size of queue.
 585          * When server is well loaded, queue size reduces to zero
 586          * after several minutes of work. It is not synflood,
 587          * it is normal operation. The solution is pruning
 588          * too old entries overriding normal timeout, when
 589          * situation becomes dangerous.
 590          *
 591          * Essentially, we reserve half of room for young
 592          * embrions; and abort old ones without pity, if old
 593          * ones are about to clog our table.
 594          */
 595         qlen = reqsk_queue_len(queue);
 596         if ((qlen << 1) > max(8U, sk_listener->sk_max_ack_backlog)) {
 597                 int young = reqsk_queue_len_young(queue) << 1;
 598
 599                 while (thresh > 2) {
 600                         if (qlen < young)
 601                                 break;
 602                         thresh--;
 603                         young <<= 1;
 604                 }
 605         }
 606         defer_accept = READ_ONCE(queue->rskq_defer_accept);
 607         if (defer_accept)
 608                 max_retries = defer_accept;
 609         syn_ack_recalc(req, thresh, max_retries, defer_accept,
 610                        &expire, &resend);
 611         req->rsk_ops->syn_ack_timeout(req);
 612         if (!expire &&
 613             (!resend ||
 614              !inet_rtx_syn_ack(sk_listener, req) ||
 615              inet_rsk(req)->acked)) {
 616                 unsigned long timeo;
 617
 618                 if (req->num_timeout++ == 0)
 619                         atomic_dec(&queue->young);
 620                 timeo = min(TCP_TIMEOUT_INIT << req->num_timeout, TCP_RTO_MAX);
 621                 mod_timer(&req->rsk_timer, jiffies + timeo);
 622                 return;
 623         }
 624 drop:
 625         inet_csk_reqsk_queue_drop_and_put(sk_listener, req);
 626 }
 627
 628 static void reqsk_queue_hash_req(struct request_sock *req,
 629                                  unsigned long timeout)
 630 {
 631         req->num_retrans = 0;
 632         req->num_timeout = 0;
 633         req->sk = NULL;
 634
 635         setup_pinned_timer(&req->rsk_timer, reqsk_timer_handler,
 636                             (unsigned long)req);
 637         mod_timer(&req->rsk_timer, jiffies + timeout);
 638
 639         inet_ehash_insert(req_to_sk(req), NULL);
 640         /* before letting lookups find us, make sure all req fields
 641          * are committed to memory and refcnt initialized.
 642          */
 643         smp_wmb();
 644         atomic_set(&req->rsk_refcnt, 2 + 1);
 645 }
 646
 647 void inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req,
 648                                    unsigned long timeout)
 649 {
 650         reqsk_queue_hash_req(req, timeout);
 651         inet_csk_reqsk_queue_added(sk);
 652 }
 653 EXPORT_SYMBOL_GPL(inet_csk_reqsk_queue_hash_add);
 654
 655 /**
 656  *      inet_csk_clone_lock - clone an inet socket, and lock its clone
 657  *      @sk: the socket to clone
 658  *      @req: request_sock
 659  *      @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
 660  *
 661  *      Caller must unlock socket even in error path (bh_unlock_sock(newsk))
 662  */
 663 struct sock *inet_csk_clone_lock(const struct sock *sk,
 664                                  const struct request_sock *req,
 665                                  const gfp_t priority)
 666 {
 667         struct sock *newsk = sk_clone_lock(sk, priority);
 668
 669         if (newsk) {
 670                 struct inet_connection_sock *newicsk = inet_csk(newsk);
 671
 672                 newsk->sk_state = TCP_SYN_RECV;
 673                 newicsk->icsk_bind_hash = NULL;
 674
 675                 inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port;
 676                 inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num;
 677                 inet_sk(newsk)->inet_sport = htons(inet_rsk(req)->ir_num);
 678                 newsk->sk_write_space = sk_stream_write_space;
 679
 680                 /* listeners have SOCK_RCU_FREE, not the children */
 681                 sock_reset_flag(newsk, SOCK_RCU_FREE);
 682
 683                 inet_sk(newsk)->mc_list = NULL;
 684
 685                 newsk->sk_mark = inet_rsk(req)->ir_mark;
 686                 atomic64_set(&newsk->sk_cookie,
 687                              atomic64_read(&inet_rsk(req)->ir_cookie));
 688
 689                 newicsk->icsk_retransmits = 0;
 690                 newicsk->icsk_backoff     = 0;
 691                 newicsk->icsk_probes_out  = 0;
 692
 693                 /* Deinitialize accept_queue to trap illegal accesses. */
 694                 memset(&newicsk->icsk_accept_queue, 0, sizeof(newicsk->icsk_accept_queue));
 695
 696                 security_inet_csk_clone(newsk, req);
 697         }
 698         return newsk;
 699 }
 700 EXPORT_SYMBOL_GPL(inet_csk_clone_lock);
 701
 702 /*
 703  * At this point, there should be no process reference to this
 704  * socket, and thus no user references at all.  Therefore we
 705  * can assume the socket waitqueue is inactive and nobody will
 706  * try to jump onto it.
 707  */
 708 void inet_csk_destroy_sock(struct sock *sk)
 709 {
 710         WARN_ON(sk->sk_state != TCP_CLOSE);
 711         WARN_ON(!sock_flag(sk, SOCK_DEAD));
 712
 713         /* It cannot be in hash table! */
 714         WARN_ON(!sk_unhashed(sk));
 715
 716         /* If it has not 0 inet_sk(sk)->inet_num, it must be bound */
 717         WARN_ON(inet_sk(sk)->inet_num && !inet_csk(sk)->icsk_bind_hash);
 718
 719         sk->sk_prot->destroy(sk);
 720
 721         sk_stream_kill_queues(sk);
 722
 723         xfrm_sk_free_policy(sk);
 724
 725         sk_refcnt_debug_release(sk);
 726
 727         local_bh_disable();
 728         percpu_counter_dec(sk->sk_prot->orphan_count);
 729         local_bh_enable();
 730         sock_put(sk);
 731 }
 732 EXPORT_SYMBOL(inet_csk_destroy_sock);
 733
 734 /* This function allows to force a closure of a socket after the call to
 735  * tcp/dccp_create_openreq_child().
 736  */
 737 void inet_csk_prepare_forced_close(struct sock *sk)
 738         __releases(&sk->sk_lock.slock)
 739 {
 740         /* sk_clone_lock locked the socket and set refcnt to 2 */
 741         bh_unlock_sock(sk);
 742         sock_put(sk);
 743
 744         /* The below has to be done to allow calling inet_csk_destroy_sock */
 745         sock_set_flag(sk, SOCK_DEAD);
 746         percpu_counter_inc(sk->sk_prot->orphan_count);
 747         inet_sk(sk)->inet_num = 0;
 748 }
 749 EXPORT_SYMBOL(inet_csk_prepare_forced_close);
 750
 751 int inet_csk_listen_start(struct sock *sk, int backlog)
 752 {
 753         struct inet_connection_sock *icsk = inet_csk(sk);
 754         struct inet_sock *inet = inet_sk(sk);
 755         int err = -EADDRINUSE;
 756
 757         reqsk_queue_alloc(&icsk->icsk_accept_queue);
 758
 759         sk->sk_max_ack_backlog = backlog;
 760         sk->sk_ack_backlog = 0;
 761         inet_csk_delack_init(sk);
 762
 763         /* There is race window here: we announce ourselves listening,
 764          * but this transition is still not validated by get_port().
 765          * It is OK, because this socket enters to hash table only
 766          * after validation is complete.
 767          */
 768         sk_state_store(sk, TCP_LISTEN);
 769         if (!sk->sk_prot->get_port(sk, inet->inet_num)) {
 770                 inet->inet_sport = htons(inet->inet_num);
 771
 772                 sk_dst_reset(sk);
 773                 err = sk->sk_prot->hash(sk);
 774
 775                 if (likely(!err))
 776                         return 0;
 777         }
 778
 779         sk->sk_state = TCP_CLOSE;
 780         return err;
 781 }
 782 EXPORT_SYMBOL_GPL(inet_csk_listen_start);
 783
 784 static void inet_child_forget(struct sock *sk, struct request_sock *req,
 785                               struct sock *child)
 786 {
 787         sk->sk_prot->disconnect(child, O_NONBLOCK);
 788
 789         sock_orphan(child);
 790
 791         percpu_counter_inc(sk->sk_prot->orphan_count);
 792
 793         if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) {
 794                 BUG_ON(tcp_sk(child)->fastopen_rsk != req);
 795                 BUG_ON(sk != req->rsk_listener);
 796
 797                 /* Paranoid, to prevent race condition if
 798                  * an inbound pkt destined for child is
 799                  * blocked by sock lock in tcp_v4_rcv().
 800                  * Also to satisfy an assertion in
 801                  * tcp_v4_destroy_sock().
 802                  */
 803                 tcp_sk(child)->fastopen_rsk = NULL;
 804         }
 805         inet_csk_destroy_sock(child);
 806 }
 807
 808 struct sock *inet_csk_reqsk_queue_add(struct sock *sk,
 809                                       struct request_sock *req,
 810                                       struct sock *child)
 811 {
 812         struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
 813
 814         spin_lock(&queue->rskq_lock);
 815         if (unlikely(sk->sk_state != TCP_LISTEN)) {
 816                 inet_child_forget(sk, req, child);
 817                 child = NULL;
 818         } else {
 819                 req->sk = child;
 820                 req->dl_next = NULL;
 821                 if (queue->rskq_accept_head == NULL)
 822                         queue->rskq_accept_head = req;
 823                 else
 824                         queue->rskq_accept_tail->dl_next = req;
 825                 queue->rskq_accept_tail = req;
 826                 sk_acceptq_added(sk);
 827         }
 828         spin_unlock(&queue->rskq_lock);
 829         return child;
 830 }
 831 EXPORT_SYMBOL(inet_csk_reqsk_queue_add);
 832
 833 struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child,
 834                                          struct request_sock *req, bool own_req)
 835 {
 836         if (own_req) {
 837                 inet_csk_reqsk_queue_drop(sk, req);
 838                 reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req);
 839                 if (inet_csk_reqsk_queue_add(sk, req, child))
 840                         return child;
 841         }
 842         /* Too bad, another child took ownership of the request, undo. */
 843         bh_unlock_sock(child);
 844         sock_put(child);
 845         return NULL;
 846 }
 847 EXPORT_SYMBOL(inet_csk_complete_hashdance);
 848
 849 /*
 850  *      This routine closes sockets which have been at least partially
 851  *      opened, but not yet accepted.
 852  */
 853 void inet_csk_listen_stop(struct sock *sk)
 854 {
 855         struct inet_connection_sock *icsk = inet_csk(sk);
 856         struct request_sock_queue *queue = &icsk->icsk_accept_queue;
 857         struct request_sock *next, *req;
 858
 859         /* Following specs, it would be better either to send FIN
 860          * (and enter FIN-WAIT-1, it is normal close)
 861          * or to send active reset (abort).
 862          * Certainly, it is pretty dangerous while synflood, but it is
 863          * bad justification for our negligence 8)
 864          * To be honest, we are not able to make either
 865          * of the variants now.                 --ANK
 866          */
 867         while ((req = reqsk_queue_remove(queue, sk)) != NULL) {
 868                 struct sock *child = req->sk;
 869
 870                 local_bh_disable();
 871                 bh_lock_sock(child);
 872                 WARN_ON(sock_owned_by_user(child));
 873                 sock_hold(child);
 874
 875                 inet_child_forget(sk, req, child);
 876                 reqsk_put(req);
 877                 bh_unlock_sock(child);
 878                 local_bh_enable();
 879                 sock_put(child);
 880
 881                 cond_resched();
 882         }
 883         if (queue->fastopenq.rskq_rst_head) {
 884                 /* Free all the reqs queued in rskq_rst_head. */
 885                 spin_lock_bh(&queue->fastopenq.lock);
 886                 req = queue->fastopenq.rskq_rst_head;
 887                 queue->fastopenq.rskq_rst_head = NULL;
 888                 spin_unlock_bh(&queue->fastopenq.lock);
 889                 while (req != NULL) {
 890                         next = req->dl_next;
 891                         reqsk_put(req);
 892                         req = next;
 893                 }
 894         }
 895         WARN_ON_ONCE(sk->sk_ack_backlog);
 896 }
 897 EXPORT_SYMBOL_GPL(inet_csk_listen_stop);
 898
 899 void inet_csk_addr2sockaddr(struct sock *sk, struct sockaddr *uaddr)
 900 {
 901         struct sockaddr_in *sin = (struct sockaddr_in *)uaddr;
 902         const struct inet_sock *inet = inet_sk(sk);
 903
 904         sin->sin_family         = AF_INET;
 905         sin->sin_addr.s_addr    = inet->inet_daddr;
 906         sin->sin_port           = inet->inet_dport;
 907 }
 908 EXPORT_SYMBOL_GPL(inet_csk_addr2sockaddr);
 909
 910 #ifdef CONFIG_COMPAT
 911 int inet_csk_compat_getsockopt(struct sock *sk, int level, int optname,
 912                                char __user *optval, int __user *optlen)
 913 {
 914         const struct inet_connection_sock *icsk = inet_csk(sk);
 915
 916         if (icsk->icsk_af_ops->compat_getsockopt)
 917                 return icsk->icsk_af_ops->compat_getsockopt(sk, level, optname,
 918                                                             optval, optlen);
 919         return icsk->icsk_af_ops->getsockopt(sk, level, optname,
 920                                              optval, optlen);
 921 }
 922 EXPORT_SYMBOL_GPL(inet_csk_compat_getsockopt);
 923
 924 int inet_csk_compat_setsockopt(struct sock *sk, int level, int optname,
 925                                char __user *optval, unsigned int optlen)
 926 {
 927         const struct inet_connection_sock *icsk = inet_csk(sk);
 928
 929         if (icsk->icsk_af_ops->compat_setsockopt)
 930                 return icsk->icsk_af_ops->compat_setsockopt(sk, level, optname,
 931                                                             optval, optlen);
 932         return icsk->icsk_af_ops->setsockopt(sk, level, optname,
 933                                              optval, optlen);
 934 }
 935 EXPORT_SYMBOL_GPL(inet_csk_compat_setsockopt);
 936 #endif
 937
 938 static struct dst_entry *inet_csk_rebuild_route(struct sock *sk, struct flowi *fl)
 939 {
 940         const struct inet_sock *inet = inet_sk(sk);
 941         const struct ip_options_rcu *inet_opt;
 942         __be32 daddr = inet->inet_daddr;
 943         struct flowi4 *fl4;
 944         struct rtable *rt;
 945
 946         rcu_read_lock();
 947         inet_opt = rcu_dereference(inet->inet_opt);
 948         if (inet_opt && inet_opt->opt.srr)
 949                 daddr = inet_opt->opt.faddr;
 950         fl4 = &fl->u.ip4;
 951         rt = ip_route_output_ports(sock_net(sk), fl4, sk, daddr,
 952                                    inet->inet_saddr, inet->inet_dport,
 953                                    inet->inet_sport, sk->sk_protocol,
 954                                    RT_CONN_FLAGS(sk), sk->sk_bound_dev_if);
 955         if (IS_ERR(rt))
 956                 rt = NULL;
 957         if (rt)
 958                 sk_setup_caps(sk, &rt->dst);
 959         rcu_read_unlock();
 960
 961         return &rt->dst;
 962 }
 963
 964 struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu)
 965 {
 966         struct dst_entry *dst = __sk_dst_check(sk, 0);
 967         struct inet_sock *inet = inet_sk(sk);
 968
 969         if (!dst) {
 970                 dst = inet_csk_rebuild_route(sk, &inet->cork.fl);
 971                 if (!dst)
 972                         goto out;
 973         }
 974         dst->ops->update_pmtu(dst, sk, NULL, mtu);
 975
 976         dst = __sk_dst_check(sk, 0);
 977         if (!dst)
 978                 dst = inet_csk_rebuild_route(sk, &inet->cork.fl);
 979 out:
 980         return dst;
 981 }
 982 EXPORT_SYMBOL_GPL(inet_csk_update_pmtu);