sys/netinet/in_pcb.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 1982, 1986, 1991, 1993, 1995
   5  *      The Regents of the University of California.
   6  * Copyright (c) 2007-2009 Robert N. M. Watson
   7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
   8  * Copyright (c) 2021-2022 Gleb Smirnoff <glebius@FreeBSD.org>
   9  * All rights reserved.
  10  *
  11  * Portions of this software were developed by Robert N. M. Watson under
  12  * contract to Juniper Networks, Inc.
  13  *
  14  * Redistribution and use in source and binary forms, with or without
  15  * modification, are permitted provided that the following conditions
  16  * are met:
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. Neither the name of the University nor the names of its contributors
  23  *    may be used to endorse or promote products derived from this software
  24  *    without specific prior written permission.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  27  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  28  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  29  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  30  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  31  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  32  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  33  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  34  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  35  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  36  * SUCH DAMAGE.
  37  *
  38  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  39  */
  40
  41 #include <sys/cdefs.h>
  42 #include "opt_ddb.h"
  43 #include "opt_ipsec.h"
  44 #include "opt_inet.h"
  45 #include "opt_inet6.h"
  46 #include "opt_ratelimit.h"
  47 #include "opt_route.h"
  48 #include "opt_rss.h"
  49
  50 #include <sys/param.h>
  51 #include <sys/hash.h>
  52 #include <sys/systm.h>
  53 #include <sys/libkern.h>
  54 #include <sys/lock.h>
  55 #include <sys/malloc.h>
  56 #include <sys/mbuf.h>
  57 #include <sys/eventhandler.h>
  58 #include <sys/domain.h>
  59 #include <sys/proc.h>
  60 #include <sys/protosw.h>
  61 #include <sys/smp.h>
  62 #include <sys/smr.h>
  63 #include <sys/socket.h>
  64 #include <sys/socketvar.h>
  65 #include <sys/sockio.h>
  66 #include <sys/priv.h>
  67 #include <sys/proc.h>
  68 #include <sys/refcount.h>
  69 #include <sys/jail.h>
  70 #include <sys/kernel.h>
  71 #include <sys/sysctl.h>
  72
  73 #ifdef DDB
  74 #include <ddb/ddb.h>
  75 #endif
  76
  77 #include <vm/uma.h>
  78 #include <vm/vm.h>
  79
  80 #include <net/if.h>
  81 #include <net/if_var.h>
  82 #include <net/if_private.h>
  83 #include <net/if_types.h>
  84 #include <net/if_llatbl.h>
  85 #include <net/route.h>
  86 #include <net/rss_config.h>
  87 #include <net/vnet.h>
  88
  89 #if defined(INET) || defined(INET6)
  90 #include <netinet/in.h>
  91 #include <netinet/in_pcb.h>
  92 #include <netinet/in_pcb_var.h>
  93 #include <netinet/tcp.h>
  94 #ifdef INET
  95 #include <netinet/in_var.h>
  96 #include <netinet/in_fib.h>
  97 #endif
  98 #include <netinet/ip_var.h>
  99 #ifdef INET6
 100 #include <netinet/ip6.h>
 101 #include <netinet6/in6_pcb.h>
 102 #include <netinet6/in6_var.h>
 103 #include <netinet6/ip6_var.h>
 104 #endif /* INET6 */
 105 #include <net/route/nhop.h>
 106 #endif
 107
 108 #include <netipsec/ipsec_support.h>
 109
 110 #include <security/mac/mac_framework.h>
 111
 112 #define INPCBLBGROUP_SIZMIN     8
 113 #define INPCBLBGROUP_SIZMAX     256
 114
 115 #define INP_FREED       0x00000200      /* Went through in_pcbfree(). */
 116 #define INP_INLBGROUP   0x01000000      /* Inserted into inpcblbgroup. */
 117
 118 /*
 119  * These configure the range of local port addresses assigned to
 120  * "unspecified" outgoing connections/packets/whatever.
 121  */
 122 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;    /* 1023 */
 123 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;    /* 600 */
 124 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;     /* 10000 */
 125 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;       /* 65535 */
 126 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;      /* 49152 */
 127 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;        /* 65535 */
 128
 129 /*
 130  * Reserved ports accessible only to root. There are significant
 131  * security considerations that must be accounted for when changing these,
 132  * but the security benefits can be great. Please be careful.
 133  */
 134 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;    /* 1023 */
 135 VNET_DEFINE(int, ipport_reservedlow);
 136
 137 /* Enable random ephemeral port allocation by default. */
 138 VNET_DEFINE(int, ipport_randomized) = 1;
 139
 140 #ifdef INET
 141 static struct inpcb     *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
 142                             struct in_addr faddr, u_int fport_arg,
 143                             struct in_addr laddr, u_int lport_arg,
 144                             int lookupflags, uint8_t numa_domain);
 145
 146 #define RANGECHK(var, min, max) \
 147         if ((var) < (min)) { (var) = (min); } \
 148         else if ((var) > (max)) { (var) = (max); }
 149
 150 static int
 151 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 152 {
 153         int error;
 154
 155         error = sysctl_handle_int(oidp, arg1, arg2, req);
 156         if (error == 0) {
 157                 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 158                 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 159                 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
 160                 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
 161                 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
 162                 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
 163         }
 164         return (error);
 165 }
 166
 167 #undef RANGECHK
 168
 169 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange,
 170     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 171     "IP Ports");
 172
 173 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
 174     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 175     &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I",
 176     "");
 177 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
 178     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 179     &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I",
 180     "");
 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
 182     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 183     &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I",
 184     "");
 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
 186     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 187     &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I",
 188     "");
 189 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
 190     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 191     &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I",
 192     "");
 193 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
 194     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 195     &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I",
 196     "");
 197 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
 198         CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
 199         &VNET_NAME(ipport_reservedhigh), 0, "");
 200 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
 201         CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
 202 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
 203         CTLFLAG_VNET | CTLFLAG_RW,
 204         &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
 205
 206 #ifdef RATELIMIT
 207 counter_u64_t rate_limit_new;
 208 counter_u64_t rate_limit_chg;
 209 counter_u64_t rate_limit_active;
 210 counter_u64_t rate_limit_alloc_fail;
 211 counter_u64_t rate_limit_set_ok;
 212
 213 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 214     "IP Rate Limiting");
 215 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD,
 216     &rate_limit_active, "Active rate limited connections");
 217 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD,
 218    &rate_limit_alloc_fail, "Rate limited connection failures");
 219 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD,
 220    &rate_limit_set_ok, "Rate limited setting succeeded");
 221 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, newrl, CTLFLAG_RD,
 222    &rate_limit_new, "Total Rate limit new attempts");
 223 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, chgrl, CTLFLAG_RD,
 224    &rate_limit_chg, "Total Rate limited change attempts");
 225 #endif /* RATELIMIT */
 226
 227 #endif /* INET */
 228
 229 VNET_DEFINE(uint32_t, in_pcbhashseed);
 230 static void
 231 in_pcbhashseed_init(void)
 232 {
 233
 234         V_in_pcbhashseed = arc4random();
 235 }
 236 VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST,
 237     in_pcbhashseed_init, 0);
 238
 239 static void in_pcbremhash(struct inpcb *);
 240
 241 /*
 242  * in_pcb.c: manage the Protocol Control Blocks.
 243  *
 244  * NOTE: It is assumed that most of these functions will be called with
 245  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
 246  * functions often modify hash chains or addresses in pcbs.
 247  */
 248
 249 static struct inpcblbgroup *
 250 in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, struct ucred *cred,
 251     u_char vflag, uint16_t port, const union in_dependaddr *addr, int size,
 252     uint8_t numa_domain)
 253 {
 254         struct inpcblbgroup *grp;
 255         size_t bytes;
 256
 257         bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
 258         grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
 259         if (grp == NULL)
 260                 return (NULL);
 261         grp->il_cred = crhold(cred);
 262         grp->il_vflag = vflag;
 263         grp->il_lport = port;
 264         grp->il_numa_domain = numa_domain;
 265         grp->il_dependladdr = *addr;
 266         grp->il_inpsiz = size;
 267         CK_LIST_INSERT_HEAD(hdr, grp, il_list);
 268         return (grp);
 269 }
 270
 271 static void
 272 in_pcblbgroup_free_deferred(epoch_context_t ctx)
 273 {
 274         struct inpcblbgroup *grp;
 275
 276         grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);
 277         crfree(grp->il_cred);
 278         free(grp, M_PCB);
 279 }
 280
 281 static void
 282 in_pcblbgroup_free(struct inpcblbgroup *grp)
 283 {
 284
 285         CK_LIST_REMOVE(grp, il_list);
 286         NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);
 287 }
 288
 289 static struct inpcblbgroup *
 290 in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
 291     struct inpcblbgroup *old_grp, int size)
 292 {
 293         struct inpcblbgroup *grp;
 294         int i;
 295
 296         grp = in_pcblbgroup_alloc(hdr, old_grp->il_cred, old_grp->il_vflag,
 297             old_grp->il_lport, &old_grp->il_dependladdr, size,
 298             old_grp->il_numa_domain);
 299         if (grp == NULL)
 300                 return (NULL);
 301
 302         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
 303             ("invalid new local group size %d and old local group count %d",
 304              grp->il_inpsiz, old_grp->il_inpcnt));
 305
 306         for (i = 0; i < old_grp->il_inpcnt; ++i)
 307                 grp->il_inp[i] = old_grp->il_inp[i];
 308         grp->il_inpcnt = old_grp->il_inpcnt;
 309         in_pcblbgroup_free(old_grp);
 310         return (grp);
 311 }
 312
 313 /*
 314  * PCB at index 'i' is removed from the group. Pull up the ones below il_inp[i]
 315  * and shrink group if possible.
 316  */
 317 static void
 318 in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, struct inpcblbgroup **grpp,
 319     int i)
 320 {
 321         struct inpcblbgroup *grp, *new_grp;
 322
 323         grp = *grpp;
 324         for (; i + 1 < grp->il_inpcnt; ++i)
 325                 grp->il_inp[i] = grp->il_inp[i + 1];
 326         grp->il_inpcnt--;
 327
 328         if (grp->il_inpsiz > INPCBLBGROUP_SIZMIN &&
 329             grp->il_inpcnt <= grp->il_inpsiz / 4) {
 330                 /* Shrink this group. */
 331                 new_grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz / 2);
 332                 if (new_grp != NULL)
 333                         *grpp = new_grp;
 334         }
 335 }
 336
 337 /*
 338  * Add PCB to load balance group for SO_REUSEPORT_LB option.
 339  */
 340 static int
 341 in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
 342 {
 343         const static struct timeval interval = { 60, 0 };
 344         static struct timeval lastprint;
 345         struct inpcbinfo *pcbinfo;
 346         struct inpcblbgrouphead *hdr;
 347         struct inpcblbgroup *grp;
 348         uint32_t idx;
 349
 350         pcbinfo = inp->inp_pcbinfo;
 351
 352         INP_WLOCK_ASSERT(inp);
 353         INP_HASH_WLOCK_ASSERT(pcbinfo);
 354
 355 #ifdef INET6
 356         /*
 357          * Don't allow IPv4 mapped INET6 wild socket.
 358          */
 359         if ((inp->inp_vflag & INP_IPV4) &&
 360             inp->inp_laddr.s_addr == INADDR_ANY &&
 361             INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
 362                 return (0);
 363         }
 364 #endif
 365
 366         idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
 367         hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
 368         CK_LIST_FOREACH(grp, hdr, il_list) {
 369                 if (grp->il_cred->cr_prison == inp->inp_cred->cr_prison &&
 370                     grp->il_vflag == inp->inp_vflag &&
 371                     grp->il_lport == inp->inp_lport &&
 372                     grp->il_numa_domain == numa_domain &&
 373                     memcmp(&grp->il_dependladdr,
 374                     &inp->inp_inc.inc_ie.ie_dependladdr,
 375                     sizeof(grp->il_dependladdr)) == 0) {
 376                         break;
 377                 }
 378         }
 379         if (grp == NULL) {
 380                 /* Create new load balance group. */
 381                 grp = in_pcblbgroup_alloc(hdr, inp->inp_cred, inp->inp_vflag,
 382                     inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
 383                     INPCBLBGROUP_SIZMIN, numa_domain);
 384                 if (grp == NULL)
 385                         return (ENOBUFS);
 386         } else if (grp->il_inpcnt == grp->il_inpsiz) {
 387                 if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
 388                         if (ratecheck(&lastprint, &interval))
 389                                 printf("lb group port %d, limit reached\n",
 390                                     ntohs(grp->il_lport));
 391                         return (0);
 392                 }
 393
 394                 /* Expand this local group. */
 395                 grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);
 396                 if (grp == NULL)
 397                         return (ENOBUFS);
 398         }
 399
 400         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
 401             ("invalid local group size %d and count %d", grp->il_inpsiz,
 402             grp->il_inpcnt));
 403
 404         grp->il_inp[grp->il_inpcnt] = inp;
 405         grp->il_inpcnt++;
 406         inp->inp_flags |= INP_INLBGROUP;
 407         return (0);
 408 }
 409
 410 /*
 411  * Remove PCB from load balance group.
 412  */
 413 static void
 414 in_pcbremlbgrouphash(struct inpcb *inp)
 415 {
 416         struct inpcbinfo *pcbinfo;
 417         struct inpcblbgrouphead *hdr;
 418         struct inpcblbgroup *grp;
 419         int i;
 420
 421         pcbinfo = inp->inp_pcbinfo;
 422
 423         INP_WLOCK_ASSERT(inp);
 424         MPASS(inp->inp_flags & INP_INLBGROUP);
 425         INP_HASH_WLOCK_ASSERT(pcbinfo);
 426
 427         hdr = &pcbinfo->ipi_lbgrouphashbase[
 428             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
 429         CK_LIST_FOREACH(grp, hdr, il_list) {
 430                 for (i = 0; i < grp->il_inpcnt; ++i) {
 431                         if (grp->il_inp[i] != inp)
 432                                 continue;
 433
 434                         if (grp->il_inpcnt == 1) {
 435                                 /* We are the last, free this local group. */
 436                                 in_pcblbgroup_free(grp);
 437                         } else {
 438                                 /* Pull up inpcbs, shrink group if possible. */
 439                                 in_pcblbgroup_reorder(hdr, &grp, i);
 440                         }
 441                         inp->inp_flags &= ~INP_INLBGROUP;
 442                         return;
 443                 }
 444         }
 445         KASSERT(0, ("%s: did not find %p", __func__, inp));
 446 }
 447
 448 int
 449 in_pcblbgroup_numa(struct inpcb *inp, int arg)
 450 {
 451         struct inpcbinfo *pcbinfo;
 452         struct inpcblbgrouphead *hdr;
 453         struct inpcblbgroup *grp;
 454         int err, i;
 455         uint8_t numa_domain;
 456
 457         switch (arg) {
 458         case TCP_REUSPORT_LB_NUMA_NODOM:
 459                 numa_domain = M_NODOM;
 460                 break;
 461         case TCP_REUSPORT_LB_NUMA_CURDOM:
 462                 numa_domain = PCPU_GET(domain);
 463                 break;
 464         default:
 465                 if (arg < 0 || arg >= vm_ndomains)
 466                         return (EINVAL);
 467                 numa_domain = arg;
 468         }
 469
 470         err = 0;
 471         pcbinfo = inp->inp_pcbinfo;
 472         INP_WLOCK_ASSERT(inp);
 473         INP_HASH_WLOCK(pcbinfo);
 474         hdr = &pcbinfo->ipi_lbgrouphashbase[
 475             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
 476         CK_LIST_FOREACH(grp, hdr, il_list) {
 477                 for (i = 0; i < grp->il_inpcnt; ++i) {
 478                         if (grp->il_inp[i] != inp)
 479                                 continue;
 480
 481                         if (grp->il_numa_domain == numa_domain) {
 482                                 goto abort_with_hash_wlock;
 483                         }
 484
 485                         /* Remove it from the old group. */
 486                         in_pcbremlbgrouphash(inp);
 487
 488                         /* Add it to the new group based on numa domain. */
 489                         in_pcbinslbgrouphash(inp, numa_domain);
 490                         goto abort_with_hash_wlock;
 491                 }
 492         }
 493         err = ENOENT;
 494 abort_with_hash_wlock:
 495         INP_HASH_WUNLOCK(pcbinfo);
 496         return (err);
 497 }
 498
 499 /* Make sure it is safe to use hashinit(9) on CK_LIST. */
 500 CTASSERT(sizeof(struct inpcbhead) == sizeof(LIST_HEAD(, inpcb)));
 501
 502 /*
 503  * Initialize an inpcbinfo - a per-VNET instance of connections db.
 504  */
 505 void
 506 in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor,
 507     u_int hash_nelements, u_int porthash_nelements)
 508 {
 509
 510         mtx_init(&pcbinfo->ipi_lock, pcbstor->ips_infolock_name, NULL, MTX_DEF);
 511         mtx_init(&pcbinfo->ipi_hash_lock, pcbstor->ips_hashlock_name,
 512             NULL, MTX_DEF);
 513 #ifdef VIMAGE
 514         pcbinfo->ipi_vnet = curvnet;
 515 #endif
 516         CK_LIST_INIT(&pcbinfo->ipi_listhead);
 517         pcbinfo->ipi_count = 0;
 518         pcbinfo->ipi_hash_exact = hashinit(hash_nelements, M_PCB,
 519             &pcbinfo->ipi_hashmask);
 520         pcbinfo->ipi_hash_wild = hashinit(hash_nelements, M_PCB,
 521             &pcbinfo->ipi_hashmask);
 522         porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1);
 523         pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
 524             &pcbinfo->ipi_porthashmask);
 525         pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,
 526             &pcbinfo->ipi_lbgrouphashmask);
 527         pcbinfo->ipi_zone = pcbstor->ips_zone;
 528         pcbinfo->ipi_portzone = pcbstor->ips_portzone;
 529         pcbinfo->ipi_smr = uma_zone_get_smr(pcbinfo->ipi_zone);
 530 }
 531
 532 /*
 533  * Destroy an inpcbinfo.
 534  */
 535 void
 536 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
 537 {
 538
 539         KASSERT(pcbinfo->ipi_count == 0,
 540             ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
 541
 542         hashdestroy(pcbinfo->ipi_hash_exact, M_PCB, pcbinfo->ipi_hashmask);
 543         hashdestroy(pcbinfo->ipi_hash_wild, M_PCB, pcbinfo->ipi_hashmask);
 544         hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
 545             pcbinfo->ipi_porthashmask);
 546         hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,
 547             pcbinfo->ipi_lbgrouphashmask);
 548         mtx_destroy(&pcbinfo->ipi_hash_lock);
 549         mtx_destroy(&pcbinfo->ipi_lock);
 550 }
 551
 552 /*
 553  * Initialize a pcbstorage - per protocol zones to allocate inpcbs.
 554  */
 555 static void inpcb_fini(void *, int);
 556 void
 557 in_pcbstorage_init(void *arg)
 558 {
 559         struct inpcbstorage *pcbstor = arg;
 560
 561         pcbstor->ips_zone = uma_zcreate(pcbstor->ips_zone_name,
 562             pcbstor->ips_size, NULL, NULL, pcbstor->ips_pcbinit,
 563             inpcb_fini, UMA_ALIGN_CACHE, UMA_ZONE_SMR);
 564         pcbstor->ips_portzone = uma_zcreate(pcbstor->ips_portzone_name,
 565             sizeof(struct inpcbport), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 566         uma_zone_set_smr(pcbstor->ips_portzone,
 567             uma_zone_get_smr(pcbstor->ips_zone));
 568 }
 569
 570 /*
 571  * Destroy a pcbstorage - used by unloadable protocols.
 572  */
 573 void
 574 in_pcbstorage_destroy(void *arg)
 575 {
 576         struct inpcbstorage *pcbstor = arg;
 577
 578         uma_zdestroy(pcbstor->ips_zone);
 579         uma_zdestroy(pcbstor->ips_portzone);
 580 }
 581
 582 /*
 583  * Allocate a PCB and associate it with the socket.
 584  * On success return with the PCB locked.
 585  */
 586 int
 587 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 588 {
 589         struct inpcb *inp;
 590 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
 591         int error;
 592 #endif
 593
 594         inp = uma_zalloc_smr(pcbinfo->ipi_zone, M_NOWAIT);
 595         if (inp == NULL)
 596                 return (ENOBUFS);
 597         bzero(&inp->inp_start_zero, inp_zero_size);
 598 #ifdef NUMA
 599         inp->inp_numa_domain = M_NODOM;
 600 #endif
 601         inp->inp_pcbinfo = pcbinfo;
 602         inp->inp_socket = so;
 603         inp->inp_cred = crhold(so->so_cred);
 604         inp->inp_inc.inc_fibnum = so->so_fibnum;
 605 #ifdef MAC
 606         error = mac_inpcb_init(inp, M_NOWAIT);
 607         if (error != 0)
 608                 goto out;
 609         mac_inpcb_create(so, inp);
 610 #endif
 611 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
 612         error = ipsec_init_pcbpolicy(inp);
 613         if (error != 0) {
 614 #ifdef MAC
 615                 mac_inpcb_destroy(inp);
 616 #endif
 617                 goto out;
 618         }
 619 #endif /*IPSEC*/
 620 #ifdef INET6
 621         if (INP_SOCKAF(so) == AF_INET6) {
 622                 inp->inp_vflag |= INP_IPV6PROTO | INP_IPV6;
 623                 if (V_ip6_v6only)
 624                         inp->inp_flags |= IN6P_IPV6_V6ONLY;
 625 #ifdef INET
 626                 else
 627                         inp->inp_vflag |= INP_IPV4;
 628 #endif
 629                 if (V_ip6_auto_flowlabel)
 630                         inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 631                 inp->in6p_hops = -1;    /* use kernel default */
 632         }
 633 #endif
 634 #if defined(INET) && defined(INET6)
 635         else
 636 #endif
 637 #ifdef INET
 638                 inp->inp_vflag |= INP_IPV4;
 639 #endif
 640         inp->inp_smr = SMR_SEQ_INVALID;
 641
 642         /*
 643          * Routes in inpcb's can cache L2 as well; they are guaranteed
 644          * to be cleaned up.
 645          */
 646         inp->inp_route.ro_flags = RT_LLE_CACHE;
 647         refcount_init(&inp->inp_refcount, 1);   /* Reference from socket. */
 648         INP_WLOCK(inp);
 649         INP_INFO_WLOCK(pcbinfo);
 650         pcbinfo->ipi_count++;
 651         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 652         CK_LIST_INSERT_HEAD(&pcbinfo->ipi_listhead, inp, inp_list);
 653         INP_INFO_WUNLOCK(pcbinfo);
 654         so->so_pcb = inp;
 655
 656         return (0);
 657
 658 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
 659 out:
 660         uma_zfree_smr(pcbinfo->ipi_zone, inp);
 661         return (error);
 662 #endif
 663 }
 664
 665 #ifdef INET
 666 int
 667 in_pcbbind(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred)
 668 {
 669         int anonport, error;
 670
 671         KASSERT(sin == NULL || sin->sin_family == AF_INET,
 672             ("%s: invalid address family for %p", __func__, sin));
 673         KASSERT(sin == NULL || sin->sin_len == sizeof(struct sockaddr_in),
 674             ("%s: invalid address length for %p", __func__, sin));
 675         INP_WLOCK_ASSERT(inp);
 676         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 677
 678         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 679                 return (EINVAL);
 680         anonport = sin == NULL || sin->sin_port == 0;
 681         error = in_pcbbind_setup(inp, sin, &inp->inp_laddr.s_addr,
 682             &inp->inp_lport, cred);
 683         if (error)
 684                 return (error);
 685         if (in_pcbinshash(inp) != 0) {
 686                 inp->inp_laddr.s_addr = INADDR_ANY;
 687                 inp->inp_lport = 0;
 688                 return (EAGAIN);
 689         }
 690         if (anonport)
 691                 inp->inp_flags |= INP_ANONPORT;
 692         return (0);
 693 }
 694 #endif
 695
 696 #if defined(INET) || defined(INET6)
 697 /*
 698  * Assign a local port like in_pcb_lport(), but also used with connect()
 699  * and a foreign address and port.  If fsa is non-NULL, choose a local port
 700  * that is unused with those, otherwise one that is completely unused.
 701  * lsa can be NULL for IPv6.
 702  */
 703 int
 704 in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp,
 705     struct sockaddr *fsa, u_short fport, struct ucred *cred, int lookupflags)
 706 {
 707         struct inpcbinfo *pcbinfo;
 708         struct inpcb *tmpinp;
 709         unsigned short *lastport;
 710         int count, error;
 711         u_short aux, first, last, lport;
 712 #ifdef INET
 713         struct in_addr laddr, faddr;
 714 #endif
 715 #ifdef INET6
 716         struct in6_addr *laddr6, *faddr6;
 717 #endif
 718
 719         pcbinfo = inp->inp_pcbinfo;
 720
 721         /*
 722          * Because no actual state changes occur here, a global write lock on
 723          * the pcbinfo isn't required.
 724          */
 725         INP_LOCK_ASSERT(inp);
 726         INP_HASH_LOCK_ASSERT(pcbinfo);
 727
 728         if (inp->inp_flags & INP_HIGHPORT) {
 729                 first = V_ipport_hifirstauto;   /* sysctl */
 730                 last  = V_ipport_hilastauto;
 731                 lastport = &pcbinfo->ipi_lasthi;
 732         } else if (inp->inp_flags & INP_LOWPORT) {
 733                 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
 734                 if (error)
 735                         return (error);
 736                 first = V_ipport_lowfirstauto;  /* 1023 */
 737                 last  = V_ipport_lowlastauto;   /* 600 */
 738                 lastport = &pcbinfo->ipi_lastlow;
 739         } else {
 740                 first = V_ipport_firstauto;     /* sysctl */
 741                 last  = V_ipport_lastauto;
 742                 lastport = &pcbinfo->ipi_lastport;
 743         }
 744
 745         /*
 746          * Instead of having two loops further down counting up or down
 747          * make sure that first is always <= last and go with only one
 748          * code path implementing all logic.
 749          */
 750         if (first > last) {
 751                 aux = first;
 752                 first = last;
 753                 last = aux;
 754         }
 755
 756 #ifdef INET
 757         laddr.s_addr = INADDR_ANY;      /* used by INET6+INET below too */
 758         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
 759                 if (lsa != NULL)
 760                         laddr = ((struct sockaddr_in *)lsa)->sin_addr;
 761                 if (fsa != NULL)
 762                         faddr = ((struct sockaddr_in *)fsa)->sin_addr;
 763         }
 764 #endif
 765 #ifdef INET6
 766         laddr6 = NULL;
 767         if ((inp->inp_vflag & INP_IPV6) != 0) {
 768                 if (lsa != NULL)
 769                         laddr6 = &((struct sockaddr_in6 *)lsa)->sin6_addr;
 770                 if (fsa != NULL)
 771                         faddr6 = &((struct sockaddr_in6 *)fsa)->sin6_addr;
 772         }
 773 #endif
 774
 775         tmpinp = NULL;
 776         lport = *lportp;
 777
 778         if (V_ipport_randomized)
 779                 *lastport = first + (arc4random() % (last - first));
 780
 781         count = last - first;
 782
 783         do {
 784                 if (count-- < 0)        /* completely used? */
 785                         return (EADDRNOTAVAIL);
 786                 ++*lastport;
 787                 if (*lastport < first || *lastport > last)
 788                         *lastport = first;
 789                 lport = htons(*lastport);
 790
 791                 if (fsa != NULL) {
 792 #ifdef INET
 793                         if (lsa->sa_family == AF_INET) {
 794                                 tmpinp = in_pcblookup_hash_locked(pcbinfo,
 795                                     faddr, fport, laddr, lport, lookupflags,
 796                                     M_NODOM);
 797                         }
 798 #endif
 799 #ifdef INET6
 800                         if (lsa->sa_family == AF_INET6) {
 801                                 tmpinp = in6_pcblookup_hash_locked(pcbinfo,
 802                                     faddr6, fport, laddr6, lport, lookupflags,
 803                                     M_NODOM);
 804                         }
 805 #endif
 806                 } else {
 807 #ifdef INET6
 808                         if ((inp->inp_vflag & INP_IPV6) != 0) {
 809                                 tmpinp = in6_pcblookup_local(pcbinfo,
 810                                     &inp->in6p_laddr, lport, lookupflags, cred);
 811 #ifdef INET
 812                                 if (tmpinp == NULL &&
 813                                     (inp->inp_vflag & INP_IPV4))
 814                                         tmpinp = in_pcblookup_local(pcbinfo,
 815                                             laddr, lport, lookupflags, cred);
 816 #endif
 817                         }
 818 #endif
 819 #if defined(INET) && defined(INET6)
 820                         else
 821 #endif
 822 #ifdef INET
 823                                 tmpinp = in_pcblookup_local(pcbinfo, laddr,
 824                                     lport, lookupflags, cred);
 825 #endif
 826                 }
 827         } while (tmpinp != NULL);
 828
 829         *lportp = lport;
 830
 831         return (0);
 832 }
 833
 834 /*
 835  * Select a local port (number) to use.
 836  */
 837 int
 838 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
 839     struct ucred *cred, int lookupflags)
 840 {
 841         struct sockaddr_in laddr;
 842
 843         if (laddrp) {
 844                 bzero(&laddr, sizeof(laddr));
 845                 laddr.sin_family = AF_INET;
 846                 laddr.sin_addr = *laddrp;
 847         }
 848         return (in_pcb_lport_dest(inp, laddrp ? (struct sockaddr *) &laddr :
 849             NULL, lportp, NULL, 0, cred, lookupflags));
 850 }
 851 #endif /* INET || INET6 */
 852
 853 #ifdef INET
 854 /*
 855  * Set up a bind operation on a PCB, performing port allocation
 856  * as required, but do not actually modify the PCB. Callers can
 857  * either complete the bind by setting inp_laddr/inp_lport and
 858  * calling in_pcbinshash(), or they can just use the resulting
 859  * port and address to authorise the sending of a once-off packet.
 860  *
 861  * On error, the values of *laddrp and *lportp are not changed.
 862  */
 863 int
 864 in_pcbbind_setup(struct inpcb *inp, struct sockaddr_in *sin, in_addr_t *laddrp,
 865     u_short *lportp, struct ucred *cred)
 866 {
 867         struct socket *so = inp->inp_socket;
 868         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 869         struct in_addr laddr;
 870         u_short lport = 0;
 871         int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
 872         int error;
 873
 874         /*
 875          * XXX: Maybe we could let SO_REUSEPORT_LB set SO_REUSEPORT bit here
 876          * so that we don't have to add to the (already messy) code below.
 877          */
 878         int reuseport_lb = (so->so_options & SO_REUSEPORT_LB);
 879
 880         /*
 881          * No state changes, so read locks are sufficient here.
 882          */
 883         INP_LOCK_ASSERT(inp);
 884         INP_HASH_LOCK_ASSERT(pcbinfo);
 885
 886         laddr.s_addr = *laddrp;
 887         if (sin != NULL && laddr.s_addr != INADDR_ANY)
 888                 return (EINVAL);
 889         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
 890                 lookupflags = INPLOOKUP_WILDCARD;
 891         if (sin == NULL) {
 892                 if ((error = prison_local_ip4(cred, &laddr)) != 0)
 893                         return (error);
 894         } else {
 895                 KASSERT(sin->sin_family == AF_INET,
 896                     ("%s: invalid family for address %p", __func__, sin));
 897                 KASSERT(sin->sin_len == sizeof(*sin),
 898                     ("%s: invalid length for address %p", __func__, sin));
 899
 900                 error = prison_local_ip4(cred, &sin->sin_addr);
 901                 if (error)
 902                         return (error);
 903                 if (sin->sin_port != *lportp) {
 904                         /* Don't allow the port to change. */
 905                         if (*lportp != 0)
 906                                 return (EINVAL);
 907                         lport = sin->sin_port;
 908                 }
 909                 /* NB: lport is left as 0 if the port isn't being changed. */
 910                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 911                         /*
 912                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 913                          * allow complete duplication of binding if
 914                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 915                          * and a multicast address is bound on both
 916                          * new and duplicated sockets.
 917                          */
 918                         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
 919                                 reuseport = SO_REUSEADDR|SO_REUSEPORT;
 920                         /*
 921                          * XXX: How to deal with SO_REUSEPORT_LB here?
 922                          * Treat same as SO_REUSEPORT for now.
 923                          */
 924                         if ((so->so_options &
 925                             (SO_REUSEADDR|SO_REUSEPORT_LB)) != 0)
 926                                 reuseport_lb = SO_REUSEADDR|SO_REUSEPORT_LB;
 927                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 928                         sin->sin_port = 0;              /* yech... */
 929                         bzero(&sin->sin_zero, sizeof(sin->sin_zero));
 930                         /*
 931                          * Is the address a local IP address?
 932                          * If INP_BINDANY is set, then the socket may be bound
 933                          * to any endpoint address, local or not.
 934                          */
 935                         if ((inp->inp_flags & INP_BINDANY) == 0 &&
 936                             ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
 937                                 return (EADDRNOTAVAIL);
 938                 }
 939                 laddr = sin->sin_addr;
 940                 if (lport) {
 941                         struct inpcb *t;
 942
 943                         /* GROSS */
 944                         if (ntohs(lport) <= V_ipport_reservedhigh &&
 945                             ntohs(lport) >= V_ipport_reservedlow &&
 946                             priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
 947                                 return (EACCES);
 948                         if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 949                             priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
 950                                 t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 951                                     lport, INPLOOKUP_WILDCARD, cred);
 952         /*
 953          * XXX
 954          * This entire block sorely needs a rewrite.
 955          */
 956                                 if (t != NULL &&
 957                                     (so->so_type != SOCK_STREAM ||
 958                                      ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
 959                                     (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
 960                                      ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
 961                                      (t->inp_socket->so_options & SO_REUSEPORT) ||
 962                                      (t->inp_socket->so_options & SO_REUSEPORT_LB) == 0) &&
 963                                     (inp->inp_cred->cr_uid !=
 964                                      t->inp_cred->cr_uid))
 965                                         return (EADDRINUSE);
 966                         }
 967                         t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 968                             lport, lookupflags, cred);
 969                         if (t != NULL && (reuseport & t->inp_socket->so_options) == 0 &&
 970                             (reuseport_lb & t->inp_socket->so_options) == 0) {
 971 #ifdef INET6
 972                                 if (ntohl(sin->sin_addr.s_addr) !=
 973                                     INADDR_ANY ||
 974                                     ntohl(t->inp_laddr.s_addr) !=
 975                                     INADDR_ANY ||
 976                                     (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
 977                                     (t->inp_vflag & INP_IPV6PROTO) == 0)
 978 #endif
 979                                                 return (EADDRINUSE);
 980                         }
 981                 }
 982         }
 983         if (*lportp != 0)
 984                 lport = *lportp;
 985         if (lport == 0) {
 986                 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
 987                 if (error != 0)
 988                         return (error);
 989         }
 990         *laddrp = laddr.s_addr;
 991         *lportp = lport;
 992         return (0);
 993 }
 994
 995 /*
 996  * Connect from a socket to a specified address.
 997  * Both address and port must be specified in argument sin.
 998  * If don't have a local address for this socket yet,
 999  * then pick one.
1000  */
1001 int
1002 in_pcbconnect(struct inpcb *inp, struct sockaddr_in *sin, struct ucred *cred,
1003     bool rehash __unused)
1004 {
1005         u_short lport, fport;
1006         in_addr_t laddr, faddr;
1007         int anonport, error;
1008
1009         INP_WLOCK_ASSERT(inp);
1010         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1011         KASSERT(in_nullhost(inp->inp_faddr),
1012             ("%s: inp is already connected", __func__));
1013
1014         lport = inp->inp_lport;
1015         laddr = inp->inp_laddr.s_addr;
1016         anonport = (lport == 0);
1017         error = in_pcbconnect_setup(inp, sin, &laddr, &lport, &faddr, &fport,
1018             cred);
1019         if (error)
1020                 return (error);
1021
1022         inp->inp_faddr.s_addr = faddr;
1023         inp->inp_fport = fport;
1024
1025         /* Do the initial binding of the local address if required. */
1026         if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
1027                 inp->inp_lport = lport;
1028                 inp->inp_laddr.s_addr = laddr;
1029                 if (in_pcbinshash(inp) != 0) {
1030                         inp->inp_laddr.s_addr = inp->inp_faddr.s_addr =
1031                             INADDR_ANY;
1032                         inp->inp_lport = inp->inp_fport = 0;
1033                         return (EAGAIN);
1034                 }
1035         } else {
1036                 inp->inp_lport = lport;
1037                 inp->inp_laddr.s_addr = laddr;
1038                 if ((inp->inp_flags & INP_INHASHLIST) != 0)
1039                         in_pcbrehash(inp);
1040                 else
1041                         in_pcbinshash(inp);
1042         }
1043
1044         if (anonport)
1045                 inp->inp_flags |= INP_ANONPORT;
1046         return (0);
1047 }
1048
1049 /*
1050  * Do proper source address selection on an unbound socket in case
1051  * of connect. Take jails into account as well.
1052  */
1053 int
1054 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
1055     struct ucred *cred)
1056 {
1057         struct ifaddr *ifa;
1058         struct sockaddr *sa;
1059         struct sockaddr_in *sin, dst;
1060         struct nhop_object *nh;
1061         int error;
1062
1063         NET_EPOCH_ASSERT();
1064         KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
1065
1066         /*
1067          * Bypass source address selection and use the primary jail IP
1068          * if requested.
1069          */
1070         if (!prison_saddrsel_ip4(cred, laddr))
1071                 return (0);
1072
1073         error = 0;
1074
1075         nh = NULL;
1076         bzero(&dst, sizeof(dst));
1077         sin = &dst;
1078         sin->sin_family = AF_INET;
1079         sin->sin_len = sizeof(struct sockaddr_in);
1080         sin->sin_addr.s_addr = faddr->s_addr;
1081
1082         /*
1083          * If route is known our src addr is taken from the i/f,
1084          * else punt.
1085          *
1086          * Find out route to destination.
1087          */
1088         if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
1089                 nh = fib4_lookup(inp->inp_inc.inc_fibnum, *faddr,
1090                     0, NHR_NONE, 0);
1091
1092         /*
1093          * If we found a route, use the address corresponding to
1094          * the outgoing interface.
1095          *
1096          * Otherwise assume faddr is reachable on a directly connected
1097          * network and try to find a corresponding interface to take
1098          * the source address from.
1099          */
1100         if (nh == NULL || nh->nh_ifp == NULL) {
1101                 struct in_ifaddr *ia;
1102                 struct ifnet *ifp;
1103
1104                 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
1105                                         inp->inp_socket->so_fibnum));
1106                 if (ia == NULL) {
1107                         ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
1108                                                 inp->inp_socket->so_fibnum));
1109                 }
1110                 if (ia == NULL) {
1111                         error = ENETUNREACH;
1112                         goto done;
1113                 }
1114
1115                 if (!prison_flag(cred, PR_IP4)) {
1116                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1117                         goto done;
1118                 }
1119
1120                 ifp = ia->ia_ifp;
1121                 ia = NULL;
1122                 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1123                         sa = ifa->ifa_addr;
1124                         if (sa->sa_family != AF_INET)
1125                                 continue;
1126                         sin = (struct sockaddr_in *)sa;
1127                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1128                                 ia = (struct in_ifaddr *)ifa;
1129                                 break;
1130                         }
1131                 }
1132                 if (ia != NULL) {
1133                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1134                         goto done;
1135                 }
1136
1137                 /* 3. As a last resort return the 'default' jail address. */
1138                 error = prison_get_ip4(cred, laddr);
1139                 goto done;
1140         }
1141
1142         /*
1143          * If the outgoing interface on the route found is not
1144          * a loopback interface, use the address from that interface.
1145          * In case of jails do those three steps:
1146          * 1. check if the interface address belongs to the jail. If so use it.
1147          * 2. check if we have any address on the outgoing interface
1148          *    belonging to this jail. If so use it.
1149          * 3. as a last resort return the 'default' jail address.
1150          */
1151         if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) {
1152                 struct in_ifaddr *ia;
1153                 struct ifnet *ifp;
1154
1155                 /* If not jailed, use the default returned. */
1156                 if (!prison_flag(cred, PR_IP4)) {
1157                         ia = (struct in_ifaddr *)nh->nh_ifa;
1158                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1159                         goto done;
1160                 }
1161
1162                 /* Jailed. */
1163                 /* 1. Check if the iface address belongs to the jail. */
1164                 sin = (struct sockaddr_in *)nh->nh_ifa->ifa_addr;
1165                 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1166                         ia = (struct in_ifaddr *)nh->nh_ifa;
1167                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1168                         goto done;
1169                 }
1170
1171                 /*
1172                  * 2. Check if we have any address on the outgoing interface
1173                  *    belonging to this jail.
1174                  */
1175                 ia = NULL;
1176                 ifp = nh->nh_ifp;
1177                 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1178                         sa = ifa->ifa_addr;
1179                         if (sa->sa_family != AF_INET)
1180                                 continue;
1181                         sin = (struct sockaddr_in *)sa;
1182                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1183                                 ia = (struct in_ifaddr *)ifa;
1184                                 break;
1185                         }
1186                 }
1187                 if (ia != NULL) {
1188                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1189                         goto done;
1190                 }
1191
1192                 /* 3. As a last resort return the 'default' jail address. */
1193                 error = prison_get_ip4(cred, laddr);
1194                 goto done;
1195         }
1196
1197         /*
1198          * The outgoing interface is marked with 'loopback net', so a route
1199          * to ourselves is here.
1200          * Try to find the interface of the destination address and then
1201          * take the address from there. That interface is not necessarily
1202          * a loopback interface.
1203          * In case of jails, check that it is an address of the jail
1204          * and if we cannot find, fall back to the 'default' jail address.
1205          */
1206         if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) {
1207                 struct in_ifaddr *ia;
1208
1209                 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&dst),
1210                                         inp->inp_socket->so_fibnum));
1211                 if (ia == NULL)
1212                         ia = ifatoia(ifa_ifwithnet(sintosa(&dst), 0,
1213                                                 inp->inp_socket->so_fibnum));
1214                 if (ia == NULL)
1215                         ia = ifatoia(ifa_ifwithaddr(sintosa(&dst)));
1216
1217                 if (!prison_flag(cred, PR_IP4)) {
1218                         if (ia == NULL) {
1219                                 error = ENETUNREACH;
1220                                 goto done;
1221                         }
1222                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1223                         goto done;
1224                 }
1225
1226                 /* Jailed. */
1227                 if (ia != NULL) {
1228                         struct ifnet *ifp;
1229
1230                         ifp = ia->ia_ifp;
1231                         ia = NULL;
1232                         CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1233                                 sa = ifa->ifa_addr;
1234                                 if (sa->sa_family != AF_INET)
1235                                         continue;
1236                                 sin = (struct sockaddr_in *)sa;
1237                                 if (prison_check_ip4(cred,
1238                                     &sin->sin_addr) == 0) {
1239                                         ia = (struct in_ifaddr *)ifa;
1240                                         break;
1241                                 }
1242                         }
1243                         if (ia != NULL) {
1244                                 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1245                                 goto done;
1246                         }
1247                 }
1248
1249                 /* 3. As a last resort return the 'default' jail address. */
1250                 error = prison_get_ip4(cred, laddr);
1251                 goto done;
1252         }
1253
1254 done:
1255         if (error == 0 && laddr->s_addr == INADDR_ANY)
1256                 return (EHOSTUNREACH);
1257         return (error);
1258 }
1259
1260 /*
1261  * Set up for a connect from a socket to the specified address.
1262  * On entry, *laddrp and *lportp should contain the current local
1263  * address and port for the PCB; these are updated to the values
1264  * that should be placed in inp_laddr and inp_lport to complete
1265  * the connect.
1266  *
1267  * On success, *faddrp and *fportp will be set to the remote address
1268  * and port. These are not updated in the error case.
1269  */
1270 int
1271 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr_in *sin,
1272     in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1273     struct ucred *cred)
1274 {
1275         struct in_ifaddr *ia;
1276         struct in_addr laddr, faddr;
1277         u_short lport, fport;
1278         int error;
1279
1280         KASSERT(sin->sin_family == AF_INET,
1281             ("%s: invalid address family for %p", __func__, sin));
1282         KASSERT(sin->sin_len == sizeof(*sin),
1283             ("%s: invalid address length for %p", __func__, sin));
1284
1285         /*
1286          * Because a global state change doesn't actually occur here, a read
1287          * lock is sufficient.
1288          */
1289         NET_EPOCH_ASSERT();
1290         INP_LOCK_ASSERT(inp);
1291         INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1292
1293         if (sin->sin_port == 0)
1294                 return (EADDRNOTAVAIL);
1295         laddr.s_addr = *laddrp;
1296         lport = *lportp;
1297         faddr = sin->sin_addr;
1298         fport = sin->sin_port;
1299 #ifdef ROUTE_MPATH
1300         if (CALC_FLOWID_OUTBOUND) {
1301                 uint32_t hash_val, hash_type;
1302
1303                 hash_val = fib4_calc_software_hash(laddr, faddr, 0, fport,
1304                     inp->inp_socket->so_proto->pr_protocol, &hash_type);
1305
1306                 inp->inp_flowid = hash_val;
1307                 inp->inp_flowtype = hash_type;
1308         }
1309 #endif
1310         if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
1311                 /*
1312                  * If the destination address is INADDR_ANY,
1313                  * use the primary local address.
1314                  * If the supplied address is INADDR_BROADCAST,
1315                  * and the primary interface supports broadcast,
1316                  * choose the broadcast address for that interface.
1317                  */
1318                 if (faddr.s_addr == INADDR_ANY) {
1319                         faddr =
1320                             IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1321                         if ((error = prison_get_ip4(cred, &faddr)) != 0)
1322                                 return (error);
1323                 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1324                         if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1325                             IFF_BROADCAST)
1326                                 faddr = satosin(&CK_STAILQ_FIRST(
1327                                     &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1328                 }
1329         }
1330         if (laddr.s_addr == INADDR_ANY) {
1331                 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1332                 /*
1333                  * If the destination address is multicast and an outgoing
1334                  * interface has been set as a multicast option, prefer the
1335                  * address of that interface as our source address.
1336                  */
1337                 if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1338                     inp->inp_moptions != NULL) {
1339                         struct ip_moptions *imo;
1340                         struct ifnet *ifp;
1341
1342                         imo = inp->inp_moptions;
1343                         if (imo->imo_multicast_ifp != NULL) {
1344                                 ifp = imo->imo_multicast_ifp;
1345                                 CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1346                                         if (ia->ia_ifp == ifp &&
1347                                             prison_check_ip4(cred,
1348                                             &ia->ia_addr.sin_addr) == 0)
1349                                                 break;
1350                                 }
1351                                 if (ia == NULL)
1352                                         error = EADDRNOTAVAIL;
1353                                 else {
1354                                         laddr = ia->ia_addr.sin_addr;
1355                                         error = 0;
1356                                 }
1357                         }
1358                 }
1359                 if (error)
1360                         return (error);
1361         }
1362
1363         if (lport != 0) {
1364                 if (in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
1365                     fport, laddr, lport, 0, M_NODOM) != NULL)
1366                         return (EADDRINUSE);
1367         } else {
1368                 struct sockaddr_in lsin, fsin;
1369
1370                 bzero(&lsin, sizeof(lsin));
1371                 bzero(&fsin, sizeof(fsin));
1372                 lsin.sin_family = AF_INET;
1373                 lsin.sin_addr = laddr;
1374                 fsin.sin_family = AF_INET;
1375                 fsin.sin_addr = faddr;
1376                 error = in_pcb_lport_dest(inp, (struct sockaddr *) &lsin,
1377                     &lport, (struct sockaddr *)& fsin, fport, cred,
1378                     INPLOOKUP_WILDCARD);
1379                 if (error)
1380                         return (error);
1381         }
1382         *laddrp = laddr.s_addr;
1383         *lportp = lport;
1384         *faddrp = faddr.s_addr;
1385         *fportp = fport;
1386         return (0);
1387 }
1388
1389 void
1390 in_pcbdisconnect(struct inpcb *inp)
1391 {
1392
1393         INP_WLOCK_ASSERT(inp);
1394         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1395         KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
1396             ("%s: inp %p was already disconnected", __func__, inp));
1397
1398         in_pcbremhash_locked(inp);
1399
1400         /* See the comment in in_pcbinshash(). */
1401         inp->inp_smr = smr_advance(inp->inp_pcbinfo->ipi_smr);
1402         inp->inp_laddr.s_addr = INADDR_ANY;
1403         inp->inp_faddr.s_addr = INADDR_ANY;
1404         inp->inp_fport = 0;
1405 }
1406 #endif /* INET */
1407
1408 /*
1409  * in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
1410  * For most protocols, this will be invoked immediately prior to calling
1411  * in_pcbfree().  However, with TCP the inpcb may significantly outlive the
1412  * socket, in which case in_pcbfree() is deferred.
1413  */
1414 void
1415 in_pcbdetach(struct inpcb *inp)
1416 {
1417
1418         KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1419
1420 #ifdef RATELIMIT
1421         if (inp->inp_snd_tag != NULL)
1422                 in_pcbdetach_txrtlmt(inp);
1423 #endif
1424         inp->inp_socket->so_pcb = NULL;
1425         inp->inp_socket = NULL;
1426 }
1427
1428 /*
1429  * inpcb hash lookups are protected by SMR section.
1430  *
1431  * Once desired pcb has been found, switching from SMR section to a pcb
1432  * lock is performed with inp_smr_lock(). We can not use INP_(W|R)LOCK
1433  * here because SMR is a critical section.
1434  * In 99%+ cases inp_smr_lock() would obtain the lock immediately.
1435  */
1436 void
1437 inp_lock(struct inpcb *inp, const inp_lookup_t lock)
1438 {
1439
1440         lock == INPLOOKUP_RLOCKPCB ?
1441             rw_rlock(&inp->inp_lock) : rw_wlock(&inp->inp_lock);
1442 }
1443
1444 void
1445 inp_unlock(struct inpcb *inp, const inp_lookup_t lock)
1446 {
1447
1448         lock == INPLOOKUP_RLOCKPCB ?
1449             rw_runlock(&inp->inp_lock) : rw_wunlock(&inp->inp_lock);
1450 }
1451
1452 int
1453 inp_trylock(struct inpcb *inp, const inp_lookup_t lock)
1454 {
1455
1456         return (lock == INPLOOKUP_RLOCKPCB ?
1457             rw_try_rlock(&inp->inp_lock) : rw_try_wlock(&inp->inp_lock));
1458 }
1459
1460 static inline bool
1461 _inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock, const int ignflags)
1462 {
1463
1464         MPASS(lock == INPLOOKUP_RLOCKPCB || lock == INPLOOKUP_WLOCKPCB);
1465         SMR_ASSERT_ENTERED(inp->inp_pcbinfo->ipi_smr);
1466
1467         if (__predict_true(inp_trylock(inp, lock))) {
1468                 if (__predict_false(inp->inp_flags & ignflags)) {
1469                         smr_exit(inp->inp_pcbinfo->ipi_smr);
1470                         inp_unlock(inp, lock);
1471                         return (false);
1472                 }
1473                 smr_exit(inp->inp_pcbinfo->ipi_smr);
1474                 return (true);
1475         }
1476
1477         if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1478                 smr_exit(inp->inp_pcbinfo->ipi_smr);
1479                 inp_lock(inp, lock);
1480                 if (__predict_false(in_pcbrele(inp, lock)))
1481                         return (false);
1482                 /*
1483                  * inp acquired through refcount & lock for sure didn't went
1484                  * through uma_zfree().  However, it may have already went
1485                  * through in_pcbfree() and has another reference, that
1486                  * prevented its release by our in_pcbrele().
1487                  */
1488                 if (__predict_false(inp->inp_flags & ignflags)) {
1489                         inp_unlock(inp, lock);
1490                         return (false);
1491                 }
1492                 return (true);
1493         } else {
1494                 smr_exit(inp->inp_pcbinfo->ipi_smr);
1495                 return (false);
1496         }
1497 }
1498
1499 bool
1500 inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock)
1501 {
1502
1503         /*
1504          * in_pcblookup() family of functions ignore not only freed entries,
1505          * that may be found due to lockless access to the hash, but dropped
1506          * entries, too.
1507          */
1508         return (_inp_smr_lock(inp, lock, INP_FREED | INP_DROPPED));
1509 }
1510
1511 /*
1512  * inp_next() - inpcb hash/list traversal iterator
1513  *
1514  * Requires initialized struct inpcb_iterator for context.
1515  * The structure can be initialized with INP_ITERATOR() or INP_ALL_ITERATOR().
1516  *
1517  * - Iterator can have either write-lock or read-lock semantics, that can not
1518  *   be changed later.
1519  * - Iterator can iterate either over all pcbs list (INP_ALL_LIST), or through
1520  *   a single hash slot.  Note: only rip_input() does the latter.
1521  * - Iterator may have optional bool matching function.  The matching function
1522  *   will be executed for each inpcb in the SMR context, so it can not acquire
1523  *   locks and can safely access only immutable fields of inpcb.
1524  *
1525  * A fresh initialized iterator has NULL inpcb in its context and that
1526  * means that inp_next() call would return the very first inpcb on the list
1527  * locked with desired semantic.  In all following calls the context pointer
1528  * shall hold the current inpcb pointer.  The KPI user is not supposed to
1529  * unlock the current inpcb!  Upon end of traversal inp_next() will return NULL
1530  * and write NULL to its context.  After end of traversal an iterator can be
1531  * reused.
1532  *
1533  * List traversals have the following features/constraints:
1534  * - New entries won't be seen, as they are always added to the head of a list.
1535  * - Removed entries won't stop traversal as long as they are not added to
1536  *   a different list. This is violated by in_pcbrehash().
1537  */
1538 #define II_LIST_FIRST(ipi, hash)                                        \
1539                 (((hash) == INP_ALL_LIST) ?                             \
1540                     CK_LIST_FIRST(&(ipi)->ipi_listhead) :               \
1541                     CK_LIST_FIRST(&(ipi)->ipi_hash_exact[(hash)]))
1542 #define II_LIST_NEXT(inp, hash)                                         \
1543                 (((hash) == INP_ALL_LIST) ?                             \
1544                     CK_LIST_NEXT((inp), inp_list) :                     \
1545                     CK_LIST_NEXT((inp), inp_hash_exact))
1546 #define II_LOCK_ASSERT(inp, lock)                                       \
1547                 rw_assert(&(inp)->inp_lock,                             \
1548                     (lock) == INPLOOKUP_RLOCKPCB ?  RA_RLOCKED : RA_WLOCKED )
1549 struct inpcb *
1550 inp_next(struct inpcb_iterator *ii)
1551 {
1552         const struct inpcbinfo *ipi = ii->ipi;
1553         inp_match_t *match = ii->match;
1554         void *ctx = ii->ctx;
1555         inp_lookup_t lock = ii->lock;
1556         int hash = ii->hash;
1557         struct inpcb *inp;
1558
1559         if (ii->inp == NULL) {          /* First call. */
1560                 smr_enter(ipi->ipi_smr);
1561                 /* This is unrolled CK_LIST_FOREACH(). */
1562                 for (inp = II_LIST_FIRST(ipi, hash);
1563                     inp != NULL;
1564                     inp = II_LIST_NEXT(inp, hash)) {
1565                         if (match != NULL && (match)(inp, ctx) == false)
1566                                 continue;
1567                         if (__predict_true(_inp_smr_lock(inp, lock, INP_FREED)))
1568                                 break;
1569                         else {
1570                                 smr_enter(ipi->ipi_smr);
1571                                 MPASS(inp != II_LIST_FIRST(ipi, hash));
1572                                 inp = II_LIST_FIRST(ipi, hash);
1573                                 if (inp == NULL)
1574                                         break;
1575                         }
1576                 }
1577
1578                 if (inp == NULL)
1579                         smr_exit(ipi->ipi_smr);
1580                 else
1581                         ii->inp = inp;
1582
1583                 return (inp);
1584         }
1585
1586         /* Not a first call. */
1587         smr_enter(ipi->ipi_smr);
1588 restart:
1589         inp = ii->inp;
1590         II_LOCK_ASSERT(inp, lock);
1591 next:
1592         inp = II_LIST_NEXT(inp, hash);
1593         if (inp == NULL) {
1594                 smr_exit(ipi->ipi_smr);
1595                 goto found;
1596         }
1597
1598         if (match != NULL && (match)(inp, ctx) == false)
1599                 goto next;
1600
1601         if (__predict_true(inp_trylock(inp, lock))) {
1602                 if (__predict_false(inp->inp_flags & INP_FREED)) {
1603                         /*
1604                          * Entries are never inserted in middle of a list, thus
1605                          * as long as we are in SMR, we can continue traversal.
1606                          * Jump to 'restart' should yield in the same result,
1607                          * but could produce unnecessary looping.  Could this
1608                          * looping be unbound?
1609                          */
1610                         inp_unlock(inp, lock);
1611                         goto next;
1612                 } else {
1613                         smr_exit(ipi->ipi_smr);
1614                         goto found;
1615                 }
1616         }
1617
1618         /*
1619          * Can't obtain lock immediately, thus going hard.  Once we exit the
1620          * SMR section we can no longer jump to 'next', and our only stable
1621          * anchoring point is ii->inp, which we keep locked for this case, so
1622          * we jump to 'restart'.
1623          */
1624         if (__predict_true(refcount_acquire_if_not_zero(&inp->inp_refcount))) {
1625                 smr_exit(ipi->ipi_smr);
1626                 inp_lock(inp, lock);
1627                 if (__predict_false(in_pcbrele(inp, lock))) {
1628                         smr_enter(ipi->ipi_smr);
1629                         goto restart;
1630                 }
1631                 /*
1632                  * See comment in inp_smr_lock().
1633                  */
1634                 if (__predict_false(inp->inp_flags & INP_FREED)) {
1635                         inp_unlock(inp, lock);
1636                         smr_enter(ipi->ipi_smr);
1637                         goto restart;
1638                 }
1639         } else
1640                 goto next;
1641
1642 found:
1643         inp_unlock(ii->inp, lock);
1644         ii->inp = inp;
1645
1646         return (ii->inp);
1647 }
1648
1649 /*
1650  * in_pcbref() bumps the reference count on an inpcb in order to maintain
1651  * stability of an inpcb pointer despite the inpcb lock being released or
1652  * SMR section exited.
1653  *
1654  * To free a reference later in_pcbrele_(r|w)locked() must be performed.
1655  */
1656 void
1657 in_pcbref(struct inpcb *inp)
1658 {
1659         u_int old __diagused;
1660
1661         old = refcount_acquire(&inp->inp_refcount);
1662         KASSERT(old > 0, ("%s: refcount 0", __func__));
1663 }
1664
1665 /*
1666  * Drop a refcount on an inpcb elevated using in_pcbref(), potentially
1667  * freeing the pcb, if the reference was very last.
1668  */
1669 bool
1670 in_pcbrele_rlocked(struct inpcb *inp)
1671 {
1672
1673         INP_RLOCK_ASSERT(inp);
1674
1675         if (!refcount_release(&inp->inp_refcount))
1676                 return (false);
1677
1678         MPASS(inp->inp_flags & INP_FREED);
1679         MPASS(inp->inp_socket == NULL);
1680         crfree(inp->inp_cred);
1681 #ifdef INVARIANTS
1682         inp->inp_cred = NULL;
1683 #endif
1684         INP_RUNLOCK(inp);
1685         uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1686         return (true);
1687 }
1688
1689 bool
1690 in_pcbrele_wlocked(struct inpcb *inp)
1691 {
1692
1693         INP_WLOCK_ASSERT(inp);
1694
1695         if (!refcount_release(&inp->inp_refcount))
1696                 return (false);
1697
1698         MPASS(inp->inp_flags & INP_FREED);
1699         MPASS(inp->inp_socket == NULL);
1700         crfree(inp->inp_cred);
1701 #ifdef INVARIANTS
1702         inp->inp_cred = NULL;
1703 #endif
1704         INP_WUNLOCK(inp);
1705         uma_zfree_smr(inp->inp_pcbinfo->ipi_zone, inp);
1706         return (true);
1707 }
1708
1709 bool
1710 in_pcbrele(struct inpcb *inp, const inp_lookup_t lock)
1711 {
1712
1713         return (lock == INPLOOKUP_RLOCKPCB ?
1714             in_pcbrele_rlocked(inp) : in_pcbrele_wlocked(inp));
1715 }
1716
1717 /*
1718  * Unconditionally schedule an inpcb to be freed by decrementing its
1719  * reference count, which should occur only after the inpcb has been detached
1720  * from its socket.  If another thread holds a temporary reference (acquired
1721  * using in_pcbref()) then the free is deferred until that reference is
1722  * released using in_pcbrele_(r|w)locked(), but the inpcb is still unlocked.
1723  *  Almost all work, including removal from global lists, is done in this
1724  * context, where the pcbinfo lock is held.
1725  */
1726 void
1727 in_pcbfree(struct inpcb *inp)
1728 {
1729         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1730 #ifdef INET
1731         struct ip_moptions *imo;
1732 #endif
1733 #ifdef INET6
1734         struct ip6_moptions *im6o;
1735 #endif
1736
1737         INP_WLOCK_ASSERT(inp);
1738         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1739         KASSERT((inp->inp_flags & INP_FREED) == 0,
1740             ("%s: called twice for pcb %p", __func__, inp));
1741
1742         inp->inp_flags |= INP_FREED;
1743         INP_INFO_WLOCK(pcbinfo);
1744         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1745         pcbinfo->ipi_count--;
1746         CK_LIST_REMOVE(inp, inp_list);
1747         INP_INFO_WUNLOCK(pcbinfo);
1748
1749         if (inp->inp_flags & INP_INHASHLIST)
1750                 in_pcbremhash(inp);
1751
1752         RO_INVALIDATE_CACHE(&inp->inp_route);
1753 #ifdef MAC
1754         mac_inpcb_destroy(inp);
1755 #endif
1756 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1757         if (inp->inp_sp != NULL)
1758                 ipsec_delete_pcbpolicy(inp);
1759 #endif
1760 #ifdef INET
1761         if (inp->inp_options)
1762                 (void)m_free(inp->inp_options);
1763         imo = inp->inp_moptions;
1764 #endif
1765 #ifdef INET6
1766         if (inp->inp_vflag & INP_IPV6PROTO) {
1767                 ip6_freepcbopts(inp->in6p_outputopts);
1768                 im6o = inp->in6p_moptions;
1769         } else
1770                 im6o = NULL;
1771 #endif
1772
1773         if (__predict_false(in_pcbrele_wlocked(inp) == false)) {
1774                 INP_WUNLOCK(inp);
1775         }
1776 #ifdef INET6
1777         ip6_freemoptions(im6o);
1778 #endif
1779 #ifdef INET
1780         inp_freemoptions(imo);
1781 #endif
1782 }
1783
1784 /*
1785  * Different protocols initialize their inpcbs differently - giving
1786  * different name to the lock.  But they all are disposed the same.
1787  */
1788 static void
1789 inpcb_fini(void *mem, int size)
1790 {
1791         struct inpcb *inp = mem;
1792
1793         INP_LOCK_DESTROY(inp);
1794 }
1795
1796 /*
1797  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1798  * port reservation, and preventing it from being returned by inpcb lookups.
1799  *
1800  * It is used by TCP to mark an inpcb as unused and avoid future packet
1801  * delivery or event notification when a socket remains open but TCP has
1802  * closed.  This might occur as a result of a shutdown()-initiated TCP close
1803  * or a RST on the wire, and allows the port binding to be reused while still
1804  * maintaining the invariant that so_pcb always points to a valid inpcb until
1805  * in_pcbdetach().
1806  *
1807  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1808  * in_pcbnotifyall() and in_pcbpurgeif0()?
1809  */
1810 void
1811 in_pcbdrop(struct inpcb *inp)
1812 {
1813
1814         INP_WLOCK_ASSERT(inp);
1815 #ifdef INVARIANTS
1816         if (inp->inp_socket != NULL && inp->inp_ppcb != NULL)
1817                 MPASS(inp->inp_refcount > 1);
1818 #endif
1819
1820         inp->inp_flags |= INP_DROPPED;
1821         if (inp->inp_flags & INP_INHASHLIST)
1822                 in_pcbremhash(inp);
1823 }
1824
1825 #ifdef INET
1826 /*
1827  * Common routines to return the socket addresses associated with inpcbs.
1828  */
1829 struct sockaddr *
1830 in_sockaddr(in_port_t port, struct in_addr *addr_p)
1831 {
1832         struct sockaddr_in *sin;
1833
1834         sin = malloc(sizeof *sin, M_SONAME,
1835                 M_WAITOK | M_ZERO);
1836         sin->sin_family = AF_INET;
1837         sin->sin_len = sizeof(*sin);
1838         sin->sin_addr = *addr_p;
1839         sin->sin_port = port;
1840
1841         return (struct sockaddr *)sin;
1842 }
1843
1844 int
1845 in_getsockaddr(struct socket *so, struct sockaddr **nam)
1846 {
1847         struct inpcb *inp;
1848         struct in_addr addr;
1849         in_port_t port;
1850
1851         inp = sotoinpcb(so);
1852         KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1853
1854         INP_RLOCK(inp);
1855         port = inp->inp_lport;
1856         addr = inp->inp_laddr;
1857         INP_RUNLOCK(inp);
1858
1859         *nam = in_sockaddr(port, &addr);
1860         return 0;
1861 }
1862
1863 int
1864 in_getpeeraddr(struct socket *so, struct sockaddr **nam)
1865 {
1866         struct inpcb *inp;
1867         struct in_addr addr;
1868         in_port_t port;
1869
1870         inp = sotoinpcb(so);
1871         KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1872
1873         INP_RLOCK(inp);
1874         port = inp->inp_fport;
1875         addr = inp->inp_faddr;
1876         INP_RUNLOCK(inp);
1877
1878         *nam = in_sockaddr(port, &addr);
1879         return 0;
1880 }
1881
1882 void
1883 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
1884     struct inpcb *(*notify)(struct inpcb *, int))
1885 {
1886         struct inpcb *inp, *inp_temp;
1887
1888         INP_INFO_WLOCK(pcbinfo);
1889         CK_LIST_FOREACH_SAFE(inp, &pcbinfo->ipi_listhead, inp_list, inp_temp) {
1890                 INP_WLOCK(inp);
1891 #ifdef INET6
1892                 if ((inp->inp_vflag & INP_IPV4) == 0) {
1893                         INP_WUNLOCK(inp);
1894                         continue;
1895                 }
1896 #endif
1897                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1898                     inp->inp_socket == NULL) {
1899                         INP_WUNLOCK(inp);
1900                         continue;
1901                 }
1902                 if ((*notify)(inp, errno))
1903                         INP_WUNLOCK(inp);
1904         }
1905         INP_INFO_WUNLOCK(pcbinfo);
1906 }
1907
1908 static bool
1909 inp_v4_multi_match(const struct inpcb *inp, void *v __unused)
1910 {
1911
1912         if ((inp->inp_vflag & INP_IPV4) && inp->inp_moptions != NULL)
1913                 return (true);
1914         else
1915                 return (false);
1916 }
1917
1918 void
1919 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1920 {
1921         struct inpcb_iterator inpi = INP_ITERATOR(pcbinfo, INPLOOKUP_WLOCKPCB,
1922             inp_v4_multi_match, NULL);
1923         struct inpcb *inp;
1924         struct in_multi *inm;
1925         struct in_mfilter *imf;
1926         struct ip_moptions *imo;
1927
1928         IN_MULTI_LOCK_ASSERT();
1929
1930         while ((inp = inp_next(&inpi)) != NULL) {
1931                 INP_WLOCK_ASSERT(inp);
1932
1933                 imo = inp->inp_moptions;
1934                 /*
1935                  * Unselect the outgoing interface if it is being
1936                  * detached.
1937                  */
1938                 if (imo->imo_multicast_ifp == ifp)
1939                         imo->imo_multicast_ifp = NULL;
1940
1941                 /*
1942                  * Drop multicast group membership if we joined
1943                  * through the interface being detached.
1944                  *
1945                  * XXX This can all be deferred to an epoch_call
1946                  */
1947 restart:
1948                 IP_MFILTER_FOREACH(imf, &imo->imo_head) {
1949                         if ((inm = imf->imf_inm) == NULL)
1950                                 continue;
1951                         if (inm->inm_ifp != ifp)
1952                                 continue;
1953                         ip_mfilter_remove(&imo->imo_head, imf);
1954                         in_leavegroup_locked(inm, NULL);
1955                         ip_mfilter_free(imf);
1956                         goto restart;
1957                 }
1958         }
1959 }
1960
1961 /*
1962  * Lookup a PCB based on the local address and port.  Caller must hold the
1963  * hash lock.  No inpcb locks or references are acquired.
1964  */
1965 #define INP_LOOKUP_MAPPED_PCB_COST      3
1966 struct inpcb *
1967 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1968     u_short lport, int lookupflags, struct ucred *cred)
1969 {
1970         struct inpcb *inp;
1971 #ifdef INET6
1972         int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1973 #else
1974         int matchwild = 3;
1975 #endif
1976         int wildcard;
1977
1978         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1979             ("%s: invalid lookup flags %d", __func__, lookupflags));
1980         INP_HASH_LOCK_ASSERT(pcbinfo);
1981
1982         if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1983                 struct inpcbhead *head;
1984                 /*
1985                  * Look for an unconnected (wildcard foreign addr) PCB that
1986                  * matches the local address and port we're looking for.
1987                  */
1988                 head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
1989                     pcbinfo->ipi_hashmask)];
1990                 CK_LIST_FOREACH(inp, head, inp_hash_wild) {
1991 #ifdef INET6
1992                         /* XXX inp locking */
1993                         if ((inp->inp_vflag & INP_IPV4) == 0)
1994                                 continue;
1995 #endif
1996                         if (inp->inp_faddr.s_addr == INADDR_ANY &&
1997                             inp->inp_laddr.s_addr == laddr.s_addr &&
1998                             inp->inp_lport == lport) {
1999                                 /*
2000                                  * Found?
2001                                  */
2002                                 if (prison_equal_ip4(cred->cr_prison,
2003                                     inp->inp_cred->cr_prison))
2004                                         return (inp);
2005                         }
2006                 }
2007                 /*
2008                  * Not found.
2009                  */
2010                 return (NULL);
2011         } else {
2012                 struct inpcbporthead *porthash;
2013                 struct inpcbport *phd;
2014                 struct inpcb *match = NULL;
2015                 /*
2016                  * Best fit PCB lookup.
2017                  *
2018                  * First see if this local port is in use by looking on the
2019                  * port hash list.
2020                  */
2021                 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
2022                     pcbinfo->ipi_porthashmask)];
2023                 CK_LIST_FOREACH(phd, porthash, phd_hash) {
2024                         if (phd->phd_port == lport)
2025                                 break;
2026                 }
2027                 if (phd != NULL) {
2028                         /*
2029                          * Port is in use by one or more PCBs. Look for best
2030                          * fit.
2031                          */
2032                         CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
2033                                 wildcard = 0;
2034                                 if (!prison_equal_ip4(inp->inp_cred->cr_prison,
2035                                     cred->cr_prison))
2036                                         continue;
2037 #ifdef INET6
2038                                 /* XXX inp locking */
2039                                 if ((inp->inp_vflag & INP_IPV4) == 0)
2040                                         continue;
2041                                 /*
2042                                  * We never select the PCB that has
2043                                  * INP_IPV6 flag and is bound to :: if
2044                                  * we have another PCB which is bound
2045                                  * to 0.0.0.0.  If a PCB has the
2046                                  * INP_IPV6 flag, then we set its cost
2047                                  * higher than IPv4 only PCBs.
2048                                  *
2049                                  * Note that the case only happens
2050                                  * when a socket is bound to ::, under
2051                                  * the condition that the use of the
2052                                  * mapped address is allowed.
2053                                  */
2054                                 if ((inp->inp_vflag & INP_IPV6) != 0)
2055                                         wildcard += INP_LOOKUP_MAPPED_PCB_COST;
2056 #endif
2057                                 if (inp->inp_faddr.s_addr != INADDR_ANY)
2058                                         wildcard++;
2059                                 if (inp->inp_laddr.s_addr != INADDR_ANY) {
2060                                         if (laddr.s_addr == INADDR_ANY)
2061                                                 wildcard++;
2062                                         else if (inp->inp_laddr.s_addr != laddr.s_addr)
2063                                                 continue;
2064                                 } else {
2065                                         if (laddr.s_addr != INADDR_ANY)
2066                                                 wildcard++;
2067                                 }
2068                                 if (wildcard < matchwild) {
2069                                         match = inp;
2070                                         matchwild = wildcard;
2071                                         if (matchwild == 0)
2072                                                 break;
2073                                 }
2074                         }
2075                 }
2076                 return (match);
2077         }
2078 }
2079 #undef INP_LOOKUP_MAPPED_PCB_COST
2080
2081 static bool
2082 in_pcblookup_lb_numa_match(const struct inpcblbgroup *grp, int domain)
2083 {
2084         return (domain == M_NODOM || domain == grp->il_numa_domain);
2085 }
2086
2087 static struct inpcb *
2088 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
2089     const struct in_addr *faddr, uint16_t fport, const struct in_addr *laddr,
2090     uint16_t lport, int domain)
2091 {
2092         const struct inpcblbgrouphead *hdr;
2093         struct inpcblbgroup *grp;
2094         struct inpcblbgroup *jail_exact, *jail_wild, *local_exact, *local_wild;
2095
2096         INP_HASH_LOCK_ASSERT(pcbinfo);
2097
2098         hdr = &pcbinfo->ipi_lbgrouphashbase[
2099             INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
2100
2101         /*
2102          * Search for an LB group match based on the following criteria:
2103          * - prefer jailed groups to non-jailed groups
2104          * - prefer exact source address matches to wildcard matches
2105          * - prefer groups bound to the specified NUMA domain
2106          */
2107         jail_exact = jail_wild = local_exact = local_wild = NULL;
2108         CK_LIST_FOREACH(grp, hdr, il_list) {
2109                 bool injail;
2110
2111 #ifdef INET6
2112                 if (!(grp->il_vflag & INP_IPV4))
2113                         continue;
2114 #endif
2115                 if (grp->il_lport != lport)
2116                         continue;
2117
2118                 injail = prison_flag(grp->il_cred, PR_IP4) != 0;
2119                 if (injail && prison_check_ip4_locked(grp->il_cred->cr_prison,
2120                     laddr) != 0)
2121                         continue;
2122
2123                 if (grp->il_laddr.s_addr == laddr->s_addr) {
2124                         if (injail) {
2125                                 jail_exact = grp;
2126                                 if (in_pcblookup_lb_numa_match(grp, domain))
2127                                         /* This is a perfect match. */
2128                                         goto out;
2129                         } else if (local_exact == NULL ||
2130                             in_pcblookup_lb_numa_match(grp, domain)) {
2131                                 local_exact = grp;
2132                         }
2133                 } else if (grp->il_laddr.s_addr == INADDR_ANY) {
2134                         if (injail) {
2135                                 if (jail_wild == NULL ||
2136                                     in_pcblookup_lb_numa_match(grp, domain))
2137                                         jail_wild = grp;
2138                         } else if (local_wild == NULL ||
2139                             in_pcblookup_lb_numa_match(grp, domain)) {
2140                                 local_wild = grp;
2141                         }
2142                 }
2143         }
2144
2145         if (jail_exact != NULL)
2146                 grp = jail_exact;
2147         else if (jail_wild != NULL)
2148                 grp = jail_wild;
2149         else if (local_exact != NULL)
2150                 grp = local_exact;
2151         else
2152                 grp = local_wild;
2153         if (grp == NULL)
2154                 return (NULL);
2155 out:
2156         return (grp->il_inp[INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) %
2157             grp->il_inpcnt]);
2158 }
2159
2160 static bool
2161 in_pcblookup_exact_match(const struct inpcb *inp, struct in_addr faddr,
2162     u_short fport, struct in_addr laddr, u_short lport)
2163 {
2164 #ifdef INET6
2165         /* XXX inp locking */
2166         if ((inp->inp_vflag & INP_IPV4) == 0)
2167                 return (false);
2168 #endif
2169         if (inp->inp_faddr.s_addr == faddr.s_addr &&
2170             inp->inp_laddr.s_addr == laddr.s_addr &&
2171             inp->inp_fport == fport &&
2172             inp->inp_lport == lport)
2173                 return (true);
2174         return (false);
2175 }
2176
2177 static struct inpcb *
2178 in_pcblookup_hash_exact(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2179     u_short fport, struct in_addr laddr, u_short lport)
2180 {
2181         struct inpcbhead *head;
2182         struct inpcb *inp;
2183
2184         INP_HASH_LOCK_ASSERT(pcbinfo);
2185
2186         head = &pcbinfo->ipi_hash_exact[INP_PCBHASH(&faddr, lport, fport,
2187             pcbinfo->ipi_hashmask)];
2188         CK_LIST_FOREACH(inp, head, inp_hash_exact) {
2189                 if (in_pcblookup_exact_match(inp, faddr, fport, laddr, lport))
2190                         return (inp);
2191         }
2192         return (NULL);
2193 }
2194
2195 typedef enum {
2196         INPLOOKUP_MATCH_NONE = 0,
2197         INPLOOKUP_MATCH_WILD = 1,
2198         INPLOOKUP_MATCH_LADDR = 2,
2199 } inp_lookup_match_t;
2200
2201 static inp_lookup_match_t
2202 in_pcblookup_wild_match(const struct inpcb *inp, struct in_addr laddr,
2203     u_short lport)
2204 {
2205 #ifdef INET6
2206         /* XXX inp locking */
2207         if ((inp->inp_vflag & INP_IPV4) == 0)
2208                 return (INPLOOKUP_MATCH_NONE);
2209 #endif
2210         if (inp->inp_faddr.s_addr != INADDR_ANY || inp->inp_lport != lport)
2211                 return (INPLOOKUP_MATCH_NONE);
2212         if (inp->inp_laddr.s_addr == INADDR_ANY)
2213                 return (INPLOOKUP_MATCH_WILD);
2214         if (inp->inp_laddr.s_addr == laddr.s_addr)
2215                 return (INPLOOKUP_MATCH_LADDR);
2216         return (INPLOOKUP_MATCH_NONE);
2217 }
2218
2219 #define INP_LOOKUP_AGAIN        ((struct inpcb *)(uintptr_t)-1)
2220
2221 static struct inpcb *
2222 in_pcblookup_hash_wild_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2223     u_short fport, struct in_addr laddr, u_short lport,
2224     const inp_lookup_t lockflags)
2225 {
2226         struct inpcbhead *head;
2227         struct inpcb *inp;
2228
2229         KASSERT(SMR_ENTERED(pcbinfo->ipi_smr),
2230             ("%s: not in SMR read section", __func__));
2231
2232         head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2233             pcbinfo->ipi_hashmask)];
2234         CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2235                 inp_lookup_match_t match;
2236
2237                 match = in_pcblookup_wild_match(inp, laddr, lport);
2238                 if (match == INPLOOKUP_MATCH_NONE)
2239                         continue;
2240
2241                 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2242                         match = in_pcblookup_wild_match(inp, laddr, lport);
2243                         if (match != INPLOOKUP_MATCH_NONE &&
2244                             prison_check_ip4_locked(inp->inp_cred->cr_prison,
2245                             &laddr) == 0)
2246                                 return (inp);
2247                         inp_unlock(inp, lockflags);
2248                 }
2249
2250                 /*
2251                  * The matching socket disappeared out from under us.  Fall back
2252                  * to a serialized lookup.
2253                  */
2254                 return (INP_LOOKUP_AGAIN);
2255         }
2256         return (NULL);
2257 }
2258
2259 static struct inpcb *
2260 in_pcblookup_hash_wild_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2261     u_short fport, struct in_addr laddr, u_short lport)
2262 {
2263         struct inpcbhead *head;
2264         struct inpcb *inp, *local_wild, *local_exact, *jail_wild;
2265 #ifdef INET6
2266         struct inpcb *local_wild_mapped;
2267 #endif
2268
2269         INP_HASH_LOCK_ASSERT(pcbinfo);
2270
2271         /*
2272          * Order of socket selection - we always prefer jails.
2273          *      1. jailed, non-wild.
2274          *      2. jailed, wild.
2275          *      3. non-jailed, non-wild.
2276          *      4. non-jailed, wild.
2277          */
2278         head = &pcbinfo->ipi_hash_wild[INP_PCBHASH_WILD(lport,
2279             pcbinfo->ipi_hashmask)];
2280         local_wild = local_exact = jail_wild = NULL;
2281 #ifdef INET6
2282         local_wild_mapped = NULL;
2283 #endif
2284         CK_LIST_FOREACH(inp, head, inp_hash_wild) {
2285                 inp_lookup_match_t match;
2286                 bool injail;
2287
2288                 match = in_pcblookup_wild_match(inp, laddr, lport);
2289                 if (match == INPLOOKUP_MATCH_NONE)
2290                         continue;
2291
2292                 injail = prison_flag(inp->inp_cred, PR_IP4) != 0;
2293                 if (injail) {
2294                         if (prison_check_ip4_locked(inp->inp_cred->cr_prison,
2295                             &laddr) != 0)
2296                                 continue;
2297                 } else {
2298                         if (local_exact != NULL)
2299                                 continue;
2300                 }
2301
2302                 if (match == INPLOOKUP_MATCH_LADDR) {
2303                         if (injail)
2304                                 return (inp);
2305                         local_exact = inp;
2306                 } else {
2307 #ifdef INET6
2308                         /* XXX inp locking, NULL check */
2309                         if (inp->inp_vflag & INP_IPV6PROTO)
2310                                 local_wild_mapped = inp;
2311                         else
2312 #endif
2313                                 if (injail)
2314                                         jail_wild = inp;
2315                                 else
2316                                         local_wild = inp;
2317                 }
2318         }
2319         if (jail_wild != NULL)
2320                 return (jail_wild);
2321         if (local_exact != NULL)
2322                 return (local_exact);
2323         if (local_wild != NULL)
2324                 return (local_wild);
2325 #ifdef INET6
2326         if (local_wild_mapped != NULL)
2327                 return (local_wild_mapped);
2328 #endif
2329         return (NULL);
2330 }
2331
2332 /*
2333  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
2334  * that the caller has either locked the hash list, which usually happens
2335  * for bind(2) operations, or is in SMR section, which happens when sorting
2336  * out incoming packets.
2337  */
2338 static struct inpcb *
2339 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2340     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2341     uint8_t numa_domain)
2342 {
2343         struct inpcb *inp;
2344         const u_short fport = fport_arg, lport = lport_arg;
2345
2346         KASSERT((lookupflags & ~INPLOOKUP_WILDCARD) == 0,
2347             ("%s: invalid lookup flags %d", __func__, lookupflags));
2348         KASSERT(faddr.s_addr != INADDR_ANY,
2349             ("%s: invalid foreign address", __func__));
2350         KASSERT(laddr.s_addr != INADDR_ANY,
2351             ("%s: invalid local address", __func__));
2352         INP_HASH_WLOCK_ASSERT(pcbinfo);
2353
2354         inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2355         if (inp != NULL)
2356                 return (inp);
2357
2358         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2359                 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2360                     &laddr, lport, numa_domain);
2361                 if (inp == NULL) {
2362                         inp = in_pcblookup_hash_wild_locked(pcbinfo, faddr,
2363                             fport, laddr, lport);
2364                 }
2365         }
2366
2367         return (inp);
2368 }
2369
2370 static struct inpcb *
2371 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2372     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2373     uint8_t numa_domain)
2374 {
2375         struct inpcb *inp;
2376         const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2377
2378         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2379             ("%s: LOCKPCB not set", __func__));
2380
2381         INP_HASH_WLOCK(pcbinfo);
2382         inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
2383             lookupflags & ~INPLOOKUP_LOCKMASK, numa_domain);
2384         if (inp != NULL && !inp_trylock(inp, lockflags)) {
2385                 in_pcbref(inp);
2386                 INP_HASH_WUNLOCK(pcbinfo);
2387                 inp_lock(inp, lockflags);
2388                 if (in_pcbrele(inp, lockflags))
2389                         /* XXX-MJ or retry until we get a negative match? */
2390                         inp = NULL;
2391         } else {
2392                 INP_HASH_WUNLOCK(pcbinfo);
2393         }
2394         return (inp);
2395 }
2396
2397 static struct inpcb *
2398 in_pcblookup_hash_smr(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2399     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2400     uint8_t numa_domain)
2401 {
2402         struct inpcb *inp;
2403         const inp_lookup_t lockflags = lookupflags & INPLOOKUP_LOCKMASK;
2404         const u_short fport = fport_arg, lport = lport_arg;
2405
2406         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2407             ("%s: invalid lookup flags %d", __func__, lookupflags));
2408         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2409             ("%s: LOCKPCB not set", __func__));
2410
2411         smr_enter(pcbinfo->ipi_smr);
2412         inp = in_pcblookup_hash_exact(pcbinfo, faddr, fport, laddr, lport);
2413         if (inp != NULL) {
2414                 if (__predict_true(inp_smr_lock(inp, lockflags))) {
2415                         /*
2416                          * Revalidate the 4-tuple, the socket could have been
2417                          * disconnected.
2418                          */
2419                         if (__predict_true(in_pcblookup_exact_match(inp,
2420                             faddr, fport, laddr, lport)))
2421                                 return (inp);
2422                         inp_unlock(inp, lockflags);
2423                 }
2424
2425                 /*
2426                  * We failed to lock the inpcb, or its connection state changed
2427                  * out from under us.  Fall back to a precise search.
2428                  */
2429                 return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2430                     lookupflags, numa_domain));
2431         }
2432
2433         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2434                 inp = in_pcblookup_lbgroup(pcbinfo, &faddr, fport,
2435                     &laddr, lport, numa_domain);
2436                 if (inp != NULL) {
2437                         if (__predict_true(inp_smr_lock(inp, lockflags))) {
2438                                 if (__predict_true(in_pcblookup_wild_match(inp,
2439                                     laddr, lport) != INPLOOKUP_MATCH_NONE))
2440                                         return (inp);
2441                                 inp_unlock(inp, lockflags);
2442                         }
2443                         inp = INP_LOOKUP_AGAIN;
2444                 } else {
2445                         inp = in_pcblookup_hash_wild_smr(pcbinfo, faddr, fport,
2446                             laddr, lport, lockflags);
2447                 }
2448                 if (inp == INP_LOOKUP_AGAIN) {
2449                         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr,
2450                             lport, lookupflags, numa_domain));
2451                 }
2452         }
2453
2454         if (inp == NULL)
2455                 smr_exit(pcbinfo->ipi_smr);
2456
2457         return (inp);
2458 }
2459
2460 /*
2461  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2462  * from which a pre-calculated hash value may be extracted.
2463  */
2464 struct inpcb *
2465 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2466     struct in_addr laddr, u_int lport, int lookupflags,
2467     struct ifnet *ifp __unused)
2468 {
2469         return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
2470             lookupflags, M_NODOM));
2471 }
2472
2473 struct inpcb *
2474 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2475     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2476     struct ifnet *ifp __unused, struct mbuf *m)
2477 {
2478         return (in_pcblookup_hash_smr(pcbinfo, faddr, fport, laddr, lport,
2479             lookupflags, m->m_pkthdr.numa_domain));
2480 }
2481 #endif /* INET */
2482
2483 static bool
2484 in_pcbjailed(const struct inpcb *inp, unsigned int flag)
2485 {
2486         return (prison_flag(inp->inp_cred, flag) != 0);
2487 }
2488
2489 /*
2490  * Insert the PCB into a hash chain using ordering rules which ensure that
2491  * in_pcblookup_hash_wild_*() always encounter the highest-ranking PCB first.
2492  *
2493  * Specifically, keep jailed PCBs in front of non-jailed PCBs, and keep PCBs
2494  * with exact local addresses ahead of wildcard PCBs.  Unbound v4-mapped v6 PCBs
2495  * always appear last no matter whether they are jailed.
2496  */
2497 static void
2498 _in_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2499 {
2500         struct inpcb *last;
2501         bool bound, injail;
2502
2503         INP_LOCK_ASSERT(inp);
2504         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2505
2506         last = NULL;
2507         bound = inp->inp_laddr.s_addr != INADDR_ANY;
2508         if (!bound && (inp->inp_vflag & INP_IPV6PROTO) != 0) {
2509                 CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2510                         if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2511                                 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2512                                 return;
2513                         }
2514                 }
2515                 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2516                 return;
2517         }
2518
2519         injail = in_pcbjailed(inp, PR_IP4);
2520         if (!injail) {
2521                 CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2522                         if (!in_pcbjailed(last, PR_IP4))
2523                                 break;
2524                         if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2525                                 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2526                                 return;
2527                         }
2528                 }
2529         } else if (!CK_LIST_EMPTY(pcbhash) &&
2530             !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP4)) {
2531                 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2532                 return;
2533         }
2534         if (!bound) {
2535                 CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
2536                         if (last->inp_laddr.s_addr == INADDR_ANY)
2537                                 break;
2538                         if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2539                                 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2540                                 return;
2541                         }
2542                 }
2543         }
2544         if (last == NULL)
2545                 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2546         else
2547                 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2548 }
2549
2550 #ifdef INET6
2551 /*
2552  * See the comment above _in_pcbinshash_wild().
2553  */
2554 static void
2555 _in6_pcbinshash_wild(struct inpcbhead *pcbhash, struct inpcb *inp)
2556 {
2557         struct inpcb *last;
2558         bool bound, injail;
2559
2560         INP_LOCK_ASSERT(inp);
2561         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2562
2563         last = NULL;
2564         bound = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr);
2565         injail = in_pcbjailed(inp, PR_IP6);
2566         if (!injail) {
2567                 CK_LIST_FOREACH(last, pcbhash, inp_hash_wild) {
2568                         if (!in_pcbjailed(last, PR_IP6))
2569                                 break;
2570                         if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2571                                 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2572                                 return;
2573                         }
2574                 }
2575         } else if (!CK_LIST_EMPTY(pcbhash) &&
2576             !in_pcbjailed(CK_LIST_FIRST(pcbhash), PR_IP6)) {
2577                 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2578                 return;
2579         }
2580         if (!bound) {
2581                 CK_LIST_FOREACH_FROM(last, pcbhash, inp_hash_wild) {
2582                         if (IN6_IS_ADDR_UNSPECIFIED(&last->in6p_laddr))
2583                                 break;
2584                         if (CK_LIST_NEXT(last, inp_hash_wild) == NULL) {
2585                                 CK_LIST_INSERT_AFTER(last, inp, inp_hash_wild);
2586                                 return;
2587                         }
2588                 }
2589         }
2590         if (last == NULL)
2591                 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_wild);
2592         else
2593                 CK_LIST_INSERT_BEFORE(last, inp, inp_hash_wild);
2594 }
2595 #endif
2596
2597 /*
2598  * Insert PCB onto various hash lists.
2599  */
2600 int
2601 in_pcbinshash(struct inpcb *inp)
2602 {
2603         struct inpcbhead *pcbhash;
2604         struct inpcbporthead *pcbporthash;
2605         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2606         struct inpcbport *phd;
2607         uint32_t hash;
2608         bool connected;
2609
2610         INP_WLOCK_ASSERT(inp);
2611         INP_HASH_WLOCK_ASSERT(pcbinfo);
2612         KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2613             ("in_pcbinshash: INP_INHASHLIST"));
2614
2615 #ifdef INET6
2616         if (inp->inp_vflag & INP_IPV6) {
2617                 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2618                     inp->inp_fport, pcbinfo->ipi_hashmask);
2619                 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2620         } else
2621 #endif
2622         {
2623                 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2624                     inp->inp_fport, pcbinfo->ipi_hashmask);
2625                 connected = !in_nullhost(inp->inp_faddr);
2626         }
2627
2628         if (connected)
2629                 pcbhash = &pcbinfo->ipi_hash_exact[hash];
2630         else
2631                 pcbhash = &pcbinfo->ipi_hash_wild[hash];
2632
2633         pcbporthash = &pcbinfo->ipi_porthashbase[
2634             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2635
2636         /*
2637          * Add entry to load balance group.
2638          * Only do this if SO_REUSEPORT_LB is set.
2639          */
2640         if ((inp->inp_socket->so_options & SO_REUSEPORT_LB) != 0) {
2641                 int error = in_pcbinslbgrouphash(inp, M_NODOM);
2642                 if (error != 0)
2643                         return (error);
2644         }
2645
2646         /*
2647          * Go through port list and look for a head for this lport.
2648          */
2649         CK_LIST_FOREACH(phd, pcbporthash, phd_hash) {
2650                 if (phd->phd_port == inp->inp_lport)
2651                         break;
2652         }
2653
2654         /*
2655          * If none exists, malloc one and tack it on.
2656          */
2657         if (phd == NULL) {
2658                 phd = uma_zalloc_smr(pcbinfo->ipi_portzone, M_NOWAIT);
2659                 if (phd == NULL) {
2660                         if ((inp->inp_flags & INP_INLBGROUP) != 0)
2661                                 in_pcbremlbgrouphash(inp);
2662                         return (ENOMEM);
2663                 }
2664                 phd->phd_port = inp->inp_lport;
2665                 CK_LIST_INIT(&phd->phd_pcblist);
2666                 CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2667         }
2668         inp->inp_phd = phd;
2669         CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2670
2671         /*
2672          * The PCB may have been disconnected in the past.  Before we can safely
2673          * make it visible in the hash table, we must wait for all readers which
2674          * may be traversing this PCB to finish.
2675          */
2676         if (inp->inp_smr != SMR_SEQ_INVALID) {
2677                 smr_wait(pcbinfo->ipi_smr, inp->inp_smr);
2678                 inp->inp_smr = SMR_SEQ_INVALID;
2679         }
2680
2681         if (connected)
2682                 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash_exact);
2683         else {
2684 #ifdef INET6
2685                 if ((inp->inp_vflag & INP_IPV6) != 0)
2686                         _in6_pcbinshash_wild(pcbhash, inp);
2687                 else
2688 #endif
2689                         _in_pcbinshash_wild(pcbhash, inp);
2690         }
2691         inp->inp_flags |= INP_INHASHLIST;
2692
2693         return (0);
2694 }
2695
2696 void
2697 in_pcbremhash_locked(struct inpcb *inp)
2698 {
2699         struct inpcbport *phd = inp->inp_phd;
2700
2701         INP_WLOCK_ASSERT(inp);
2702         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
2703         MPASS(inp->inp_flags & INP_INHASHLIST);
2704
2705         if ((inp->inp_flags & INP_INLBGROUP) != 0)
2706                 in_pcbremlbgrouphash(inp);
2707 #ifdef INET6
2708         if (inp->inp_vflag & INP_IPV6) {
2709                 if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
2710                         CK_LIST_REMOVE(inp, inp_hash_wild);
2711                 else
2712                         CK_LIST_REMOVE(inp, inp_hash_exact);
2713         } else
2714 #endif
2715         {
2716                 if (in_nullhost(inp->inp_faddr))
2717                         CK_LIST_REMOVE(inp, inp_hash_wild);
2718                 else
2719                         CK_LIST_REMOVE(inp, inp_hash_exact);
2720         }
2721         CK_LIST_REMOVE(inp, inp_portlist);
2722         if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
2723                 CK_LIST_REMOVE(phd, phd_hash);
2724                 uma_zfree_smr(inp->inp_pcbinfo->ipi_portzone, phd);
2725         }
2726         inp->inp_flags &= ~INP_INHASHLIST;
2727 }
2728
2729 static void
2730 in_pcbremhash(struct inpcb *inp)
2731 {
2732         INP_HASH_WLOCK(inp->inp_pcbinfo);
2733         in_pcbremhash_locked(inp);
2734         INP_HASH_WUNLOCK(inp->inp_pcbinfo);
2735 }
2736
2737 /*
2738  * Move PCB to the proper hash bucket when { faddr, fport } have  been
2739  * changed. NOTE: This does not handle the case of the lport changing (the
2740  * hashed port list would have to be updated as well), so the lport must
2741  * not change after in_pcbinshash() has been called.
2742  */
2743 void
2744 in_pcbrehash(struct inpcb *inp)
2745 {
2746         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2747         struct inpcbhead *head;
2748         uint32_t hash;
2749         bool connected;
2750
2751         INP_WLOCK_ASSERT(inp);
2752         INP_HASH_WLOCK_ASSERT(pcbinfo);
2753         KASSERT(inp->inp_flags & INP_INHASHLIST,
2754             ("%s: !INP_INHASHLIST", __func__));
2755         KASSERT(inp->inp_smr == SMR_SEQ_INVALID,
2756             ("%s: inp was disconnected", __func__));
2757
2758 #ifdef INET6
2759         if (inp->inp_vflag & INP_IPV6) {
2760                 hash = INP6_PCBHASH(&inp->in6p_faddr, inp->inp_lport,
2761                     inp->inp_fport, pcbinfo->ipi_hashmask);
2762                 connected = !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr);
2763         } else
2764 #endif
2765         {
2766                 hash = INP_PCBHASH(&inp->inp_faddr, inp->inp_lport,
2767                     inp->inp_fport, pcbinfo->ipi_hashmask);
2768                 connected = !in_nullhost(inp->inp_faddr);
2769         }
2770
2771         /*
2772          * When rehashing, the caller must ensure that either the new or the old
2773          * foreign address was unspecified.
2774          */
2775         if (connected)
2776                 CK_LIST_REMOVE(inp, inp_hash_wild);
2777         else
2778                 CK_LIST_REMOVE(inp, inp_hash_exact);
2779
2780         if (connected) {
2781                 head = &pcbinfo->ipi_hash_exact[hash];
2782                 CK_LIST_INSERT_HEAD(head, inp, inp_hash_exact);
2783         } else {
2784                 head = &pcbinfo->ipi_hash_wild[hash];
2785                 CK_LIST_INSERT_HEAD(head, inp, inp_hash_wild);
2786         }
2787 }
2788
2789 /*
2790  * Check for alternatives when higher level complains
2791  * about service problems.  For now, invalidate cached
2792  * routing information.  If the route was created dynamically
2793  * (by a redirect), time to try a default gateway again.
2794  */
2795 void
2796 in_losing(struct inpcb *inp)
2797 {
2798
2799         RO_INVALIDATE_CACHE(&inp->inp_route);
2800         return;
2801 }
2802
2803 /*
2804  * A set label operation has occurred at the socket layer, propagate the
2805  * label change into the in_pcb for the socket.
2806  */
2807 void
2808 in_pcbsosetlabel(struct socket *so)
2809 {
2810 #ifdef MAC
2811         struct inpcb *inp;
2812
2813         inp = sotoinpcb(so);
2814         KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2815
2816         INP_WLOCK(inp);
2817         SOCK_LOCK(so);
2818         mac_inpcb_sosetlabel(so, inp);
2819         SOCK_UNLOCK(so);
2820         INP_WUNLOCK(inp);
2821 #endif
2822 }
2823
2824 void
2825 inp_wlock(struct inpcb *inp)
2826 {
2827
2828         INP_WLOCK(inp);
2829 }
2830
2831 void
2832 inp_wunlock(struct inpcb *inp)
2833 {
2834
2835         INP_WUNLOCK(inp);
2836 }
2837
2838 void
2839 inp_rlock(struct inpcb *inp)
2840 {
2841
2842         INP_RLOCK(inp);
2843 }
2844
2845 void
2846 inp_runlock(struct inpcb *inp)
2847 {
2848
2849         INP_RUNLOCK(inp);
2850 }
2851
2852 #ifdef INVARIANT_SUPPORT
2853 void
2854 inp_lock_assert(struct inpcb *inp)
2855 {
2856
2857         INP_WLOCK_ASSERT(inp);
2858 }
2859
2860 void
2861 inp_unlock_assert(struct inpcb *inp)
2862 {
2863
2864         INP_UNLOCK_ASSERT(inp);
2865 }
2866 #endif
2867
2868 void
2869 inp_apply_all(struct inpcbinfo *pcbinfo,
2870     void (*func)(struct inpcb *, void *), void *arg)
2871 {
2872         struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
2873             INPLOOKUP_WLOCKPCB);
2874         struct inpcb *inp;
2875
2876         while ((inp = inp_next(&inpi)) != NULL)
2877                 func(inp, arg);
2878 }
2879
2880 struct socket *
2881 inp_inpcbtosocket(struct inpcb *inp)
2882 {
2883
2884         INP_WLOCK_ASSERT(inp);
2885         return (inp->inp_socket);
2886 }
2887
2888 struct tcpcb *
2889 inp_inpcbtotcpcb(struct inpcb *inp)
2890 {
2891
2892         INP_WLOCK_ASSERT(inp);
2893         return ((struct tcpcb *)inp->inp_ppcb);
2894 }
2895
2896 int
2897 inp_ip_tos_get(const struct inpcb *inp)
2898 {
2899
2900         return (inp->inp_ip_tos);
2901 }
2902
2903 void
2904 inp_ip_tos_set(struct inpcb *inp, int val)
2905 {
2906
2907         inp->inp_ip_tos = val;
2908 }
2909
2910 void
2911 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2912     uint32_t *faddr, uint16_t *fp)
2913 {
2914
2915         INP_LOCK_ASSERT(inp);
2916         *laddr = inp->inp_laddr.s_addr;
2917         *faddr = inp->inp_faddr.s_addr;
2918         *lp = inp->inp_lport;
2919         *fp = inp->inp_fport;
2920 }
2921
2922 struct inpcb *
2923 so_sotoinpcb(struct socket *so)
2924 {
2925
2926         return (sotoinpcb(so));
2927 }
2928
2929 /*
2930  * Create an external-format (``xinpcb'') structure using the information in
2931  * the kernel-format in_pcb structure pointed to by inp.  This is done to
2932  * reduce the spew of irrelevant information over this interface, to isolate
2933  * user code from changes in the kernel structure, and potentially to provide
2934  * information-hiding if we decide that some of this information should be
2935  * hidden from users.
2936  */
2937 void
2938 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2939 {
2940
2941         bzero(xi, sizeof(*xi));
2942         xi->xi_len = sizeof(struct xinpcb);
2943         if (inp->inp_socket)
2944                 sotoxsocket(inp->inp_socket, &xi->xi_socket);
2945         bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2946         xi->inp_gencnt = inp->inp_gencnt;
2947         xi->inp_ppcb = (uintptr_t)inp->inp_ppcb;
2948         xi->inp_flow = inp->inp_flow;
2949         xi->inp_flowid = inp->inp_flowid;
2950         xi->inp_flowtype = inp->inp_flowtype;
2951         xi->inp_flags = inp->inp_flags;
2952         xi->inp_flags2 = inp->inp_flags2;
2953         xi->in6p_cksum = inp->in6p_cksum;
2954         xi->in6p_hops = inp->in6p_hops;
2955         xi->inp_ip_tos = inp->inp_ip_tos;
2956         xi->inp_vflag = inp->inp_vflag;
2957         xi->inp_ip_ttl = inp->inp_ip_ttl;
2958         xi->inp_ip_p = inp->inp_ip_p;
2959         xi->inp_ip_minttl = inp->inp_ip_minttl;
2960 }
2961
2962 int
2963 sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo,
2964     int (*ctloutput_set)(struct inpcb *, struct sockopt *))
2965 {
2966         struct sockopt sopt;
2967         struct inpcb_iterator inpi = INP_ALL_ITERATOR(pcbinfo,
2968             INPLOOKUP_WLOCKPCB);
2969         struct inpcb *inp;
2970         struct sockopt_parameters *params;
2971         struct socket *so;
2972         int error;
2973         char buf[1024];
2974
2975         if (req->oldptr != NULL || req->oldlen != 0)
2976                 return (EINVAL);
2977         if (req->newptr == NULL)
2978                 return (EPERM);
2979         if (req->newlen > sizeof(buf))
2980                 return (ENOMEM);
2981         error = SYSCTL_IN(req, buf, req->newlen);
2982         if (error != 0)
2983                 return (error);
2984         if (req->newlen < sizeof(struct sockopt_parameters))
2985                 return (EINVAL);
2986         params = (struct sockopt_parameters *)buf;
2987         sopt.sopt_level = params->sop_level;
2988         sopt.sopt_name = params->sop_optname;
2989         sopt.sopt_dir = SOPT_SET;
2990         sopt.sopt_val = params->sop_optval;
2991         sopt.sopt_valsize = req->newlen - sizeof(struct sockopt_parameters);
2992         sopt.sopt_td = NULL;
2993 #ifdef INET6
2994         if (params->sop_inc.inc_flags & INC_ISIPV6) {
2995                 if (IN6_IS_SCOPE_LINKLOCAL(&params->sop_inc.inc6_laddr))
2996                         params->sop_inc.inc6_laddr.s6_addr16[1] =
2997                             htons(params->sop_inc.inc6_zoneid & 0xffff);
2998                 if (IN6_IS_SCOPE_LINKLOCAL(&params->sop_inc.inc6_faddr))
2999                         params->sop_inc.inc6_faddr.s6_addr16[1] =
3000                             htons(params->sop_inc.inc6_zoneid & 0xffff);
3001         }
3002 #endif
3003         if (params->sop_inc.inc_lport != htons(0)) {
3004                 if (params->sop_inc.inc_fport == htons(0))
3005                         inpi.hash = INP_PCBHASH_WILD(params->sop_inc.inc_lport,
3006                             pcbinfo->ipi_hashmask);
3007                 else
3008 #ifdef INET6
3009                         if (params->sop_inc.inc_flags & INC_ISIPV6)
3010                                 inpi.hash = INP6_PCBHASH(
3011                                     &params->sop_inc.inc6_faddr,
3012                                     params->sop_inc.inc_lport,
3013                                     params->sop_inc.inc_fport,
3014                                     pcbinfo->ipi_hashmask);
3015                         else
3016 #endif
3017                                 inpi.hash = INP_PCBHASH(
3018                                     &params->sop_inc.inc_faddr,
3019                                     params->sop_inc.inc_lport,
3020                                     params->sop_inc.inc_fport,
3021                                     pcbinfo->ipi_hashmask);
3022         }
3023         while ((inp = inp_next(&inpi)) != NULL)
3024                 if (inp->inp_gencnt == params->sop_id) {
3025                         if (inp->inp_flags & INP_DROPPED) {
3026                                 INP_WUNLOCK(inp);
3027                                 return (ECONNRESET);
3028                         }
3029                         so = inp->inp_socket;
3030                         KASSERT(so != NULL, ("inp_socket == NULL"));
3031                         soref(so);
3032                         error = (*ctloutput_set)(inp, &sopt);
3033                         sorele(so);
3034                         break;
3035                 }
3036         if (inp == NULL)
3037                 error = ESRCH;
3038         return (error);
3039 }
3040
3041 #ifdef DDB
3042 static void
3043 db_print_indent(int indent)
3044 {
3045         int i;
3046
3047         for (i = 0; i < indent; i++)
3048                 db_printf(" ");
3049 }
3050
3051 static void
3052 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
3053 {
3054         char faddr_str[48], laddr_str[48];
3055
3056         db_print_indent(indent);
3057         db_printf("%s at %p\n", name, inc);
3058
3059         indent += 2;
3060
3061 #ifdef INET6
3062         if (inc->inc_flags & INC_ISIPV6) {
3063                 /* IPv6. */
3064                 ip6_sprintf(laddr_str, &inc->inc6_laddr);
3065                 ip6_sprintf(faddr_str, &inc->inc6_faddr);
3066         } else
3067 #endif
3068         {
3069                 /* IPv4. */
3070                 inet_ntoa_r(inc->inc_laddr, laddr_str);
3071                 inet_ntoa_r(inc->inc_faddr, faddr_str);
3072         }
3073         db_print_indent(indent);
3074         db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
3075             ntohs(inc->inc_lport));
3076         db_print_indent(indent);
3077         db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
3078             ntohs(inc->inc_fport));
3079 }
3080
3081 static void
3082 db_print_inpflags(int inp_flags)
3083 {
3084         int comma;
3085
3086         comma = 0;
3087         if (inp_flags & INP_RECVOPTS) {
3088                 db_printf("%sINP_RECVOPTS", comma ? ", " : "");
3089                 comma = 1;
3090         }
3091         if (inp_flags & INP_RECVRETOPTS) {
3092                 db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
3093                 comma = 1;
3094         }
3095         if (inp_flags & INP_RECVDSTADDR) {
3096                 db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
3097                 comma = 1;
3098         }
3099         if (inp_flags & INP_ORIGDSTADDR) {
3100                 db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
3101                 comma = 1;
3102         }
3103         if (inp_flags & INP_HDRINCL) {
3104                 db_printf("%sINP_HDRINCL", comma ? ", " : "");
3105                 comma = 1;
3106         }
3107         if (inp_flags & INP_HIGHPORT) {
3108                 db_printf("%sINP_HIGHPORT", comma ? ", " : "");
3109                 comma = 1;
3110         }
3111         if (inp_flags & INP_LOWPORT) {
3112                 db_printf("%sINP_LOWPORT", comma ? ", " : "");
3113                 comma = 1;
3114         }
3115         if (inp_flags & INP_ANONPORT) {
3116                 db_printf("%sINP_ANONPORT", comma ? ", " : "");
3117                 comma = 1;
3118         }
3119         if (inp_flags & INP_RECVIF) {
3120                 db_printf("%sINP_RECVIF", comma ? ", " : "");
3121                 comma = 1;
3122         }
3123         if (inp_flags & INP_MTUDISC) {
3124                 db_printf("%sINP_MTUDISC", comma ? ", " : "");
3125                 comma = 1;
3126         }
3127         if (inp_flags & INP_RECVTTL) {
3128                 db_printf("%sINP_RECVTTL", comma ? ", " : "");
3129                 comma = 1;
3130         }
3131         if (inp_flags & INP_DONTFRAG) {
3132                 db_printf("%sINP_DONTFRAG", comma ? ", " : "");
3133                 comma = 1;
3134         }
3135         if (inp_flags & INP_RECVTOS) {
3136                 db_printf("%sINP_RECVTOS", comma ? ", " : "");
3137                 comma = 1;
3138         }
3139         if (inp_flags & IN6P_IPV6_V6ONLY) {
3140                 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
3141                 comma = 1;
3142         }
3143         if (inp_flags & IN6P_PKTINFO) {
3144                 db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
3145                 comma = 1;
3146         }
3147         if (inp_flags & IN6P_HOPLIMIT) {
3148                 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
3149                 comma = 1;
3150         }
3151         if (inp_flags & IN6P_HOPOPTS) {
3152                 db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
3153                 comma = 1;
3154         }
3155         if (inp_flags & IN6P_DSTOPTS) {
3156                 db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
3157                 comma = 1;
3158         }
3159         if (inp_flags & IN6P_RTHDR) {
3160                 db_printf("%sIN6P_RTHDR", comma ? ", " : "");
3161                 comma = 1;
3162         }
3163         if (inp_flags & IN6P_RTHDRDSTOPTS) {
3164                 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
3165                 comma = 1;
3166         }
3167         if (inp_flags & IN6P_TCLASS) {
3168                 db_printf("%sIN6P_TCLASS", comma ? ", " : "");
3169                 comma = 1;
3170         }
3171         if (inp_flags & IN6P_AUTOFLOWLABEL) {
3172                 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
3173                 comma = 1;
3174         }
3175         if (inp_flags & INP_ONESBCAST) {
3176                 db_printf("%sINP_ONESBCAST", comma ? ", " : "");
3177                 comma  = 1;
3178         }
3179         if (inp_flags & INP_DROPPED) {
3180                 db_printf("%sINP_DROPPED", comma ? ", " : "");
3181                 comma  = 1;
3182         }
3183         if (inp_flags & INP_SOCKREF) {
3184                 db_printf("%sINP_SOCKREF", comma ? ", " : "");
3185                 comma  = 1;
3186         }
3187         if (inp_flags & IN6P_RFC2292) {
3188                 db_printf("%sIN6P_RFC2292", comma ? ", " : "");
3189                 comma = 1;
3190         }
3191         if (inp_flags & IN6P_MTU) {
3192                 db_printf("IN6P_MTU%s", comma ? ", " : "");
3193                 comma = 1;
3194         }
3195 }
3196
3197 static void
3198 db_print_inpvflag(u_char inp_vflag)
3199 {
3200         int comma;
3201
3202         comma = 0;
3203         if (inp_vflag & INP_IPV4) {
3204                 db_printf("%sINP_IPV4", comma ? ", " : "");
3205                 comma  = 1;
3206         }
3207         if (inp_vflag & INP_IPV6) {
3208                 db_printf("%sINP_IPV6", comma ? ", " : "");
3209                 comma  = 1;
3210         }
3211         if (inp_vflag & INP_IPV6PROTO) {
3212                 db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
3213                 comma  = 1;
3214         }
3215 }
3216
3217 static void
3218 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
3219 {
3220
3221         db_print_indent(indent);
3222         db_printf("%s at %p\n", name, inp);
3223
3224         indent += 2;
3225
3226         db_print_indent(indent);
3227         db_printf("inp_flow: 0x%x\n", inp->inp_flow);
3228
3229         db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
3230
3231         db_print_indent(indent);
3232         db_printf("inp_ppcb: %p   inp_pcbinfo: %p   inp_socket: %p\n",
3233             inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
3234
3235         db_print_indent(indent);
3236         db_printf("inp_label: %p   inp_flags: 0x%x (",
3237            inp->inp_label, inp->inp_flags);
3238         db_print_inpflags(inp->inp_flags);
3239         db_printf(")\n");
3240
3241         db_print_indent(indent);
3242         db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
3243             inp->inp_vflag);
3244         db_print_inpvflag(inp->inp_vflag);
3245         db_printf(")\n");
3246
3247         db_print_indent(indent);
3248         db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
3249             inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
3250
3251         db_print_indent(indent);
3252 #ifdef INET6
3253         if (inp->inp_vflag & INP_IPV6) {
3254                 db_printf("in6p_options: %p   in6p_outputopts: %p   "
3255                     "in6p_moptions: %p\n", inp->in6p_options,
3256                     inp->in6p_outputopts, inp->in6p_moptions);
3257                 db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
3258                     "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
3259                     inp->in6p_hops);
3260         } else
3261 #endif
3262         {
3263                 db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
3264                     "inp_ip_moptions: %p\n", inp->inp_ip_tos,
3265                     inp->inp_options, inp->inp_moptions);
3266         }
3267
3268         db_print_indent(indent);
3269         db_printf("inp_phd: %p   inp_gencnt: %ju\n", inp->inp_phd,
3270             (uintmax_t)inp->inp_gencnt);
3271 }
3272
3273 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
3274 {
3275         struct inpcb *inp;
3276
3277         if (!have_addr) {
3278                 db_printf("usage: show inpcb <addr>\n");
3279                 return;
3280         }
3281         inp = (struct inpcb *)addr;
3282
3283         db_print_inpcb(inp, "inpcb", 0);
3284 }
3285 #endif /* DDB */
3286
3287 #ifdef RATELIMIT
3288 /*
3289  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3290  * if any.
3291  */
3292 int
3293 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
3294 {
3295         union if_snd_tag_modify_params params = {
3296                 .rate_limit.max_rate = max_pacing_rate,
3297                 .rate_limit.flags = M_NOWAIT,
3298         };
3299         struct m_snd_tag *mst;
3300         int error;
3301
3302         mst = inp->inp_snd_tag;
3303         if (mst == NULL)
3304                 return (EINVAL);
3305
3306         if (mst->sw->snd_tag_modify == NULL) {
3307                 error = EOPNOTSUPP;
3308         } else {
3309                 error = mst->sw->snd_tag_modify(mst, &params);
3310         }
3311         return (error);
3312 }
3313
3314 /*
3315  * Query existing TX rate limit based on the existing
3316  * "inp->inp_snd_tag", if any.
3317  */
3318 int
3319 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
3320 {
3321         union if_snd_tag_query_params params = { };
3322         struct m_snd_tag *mst;
3323         int error;
3324
3325         mst = inp->inp_snd_tag;
3326         if (mst == NULL)
3327                 return (EINVAL);
3328
3329         if (mst->sw->snd_tag_query == NULL) {
3330                 error = EOPNOTSUPP;
3331         } else {
3332                 error = mst->sw->snd_tag_query(mst, &params);
3333                 if (error == 0 && p_max_pacing_rate != NULL)
3334                         *p_max_pacing_rate = params.rate_limit.max_rate;
3335         }
3336         return (error);
3337 }
3338
3339 /*
3340  * Query existing TX queue level based on the existing
3341  * "inp->inp_snd_tag", if any.
3342  */
3343 int
3344 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
3345 {
3346         union if_snd_tag_query_params params = { };
3347         struct m_snd_tag *mst;
3348         int error;
3349
3350         mst = inp->inp_snd_tag;
3351         if (mst == NULL)
3352                 return (EINVAL);
3353
3354         if (mst->sw->snd_tag_query == NULL)
3355                 return (EOPNOTSUPP);
3356
3357         error = mst->sw->snd_tag_query(mst, &params);
3358         if (error == 0 && p_txqueue_level != NULL)
3359                 *p_txqueue_level = params.rate_limit.queue_level;
3360         return (error);
3361 }
3362
3363 /*
3364  * Allocate a new TX rate limit send tag from the network interface
3365  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3366  */
3367 int
3368 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
3369     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st)
3370
3371 {
3372         union if_snd_tag_alloc_params params = {
3373                 .rate_limit.hdr.type = (max_pacing_rate == -1U) ?
3374                     IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
3375                 .rate_limit.hdr.flowid = flowid,
3376                 .rate_limit.hdr.flowtype = flowtype,
3377                 .rate_limit.hdr.numa_domain = inp->inp_numa_domain,
3378                 .rate_limit.max_rate = max_pacing_rate,
3379                 .rate_limit.flags = M_NOWAIT,
3380         };
3381         int error;
3382
3383         INP_WLOCK_ASSERT(inp);
3384
3385         /*
3386          * If there is already a send tag, or the INP is being torn
3387          * down, allocating a new send tag is not allowed. Else send
3388          * tags may leak.
3389          */
3390         if (*st != NULL || (inp->inp_flags & INP_DROPPED) != 0)
3391                 return (EINVAL);
3392
3393         error = m_snd_tag_alloc(ifp, &params, st);
3394 #ifdef INET
3395         if (error == 0) {
3396                 counter_u64_add(rate_limit_set_ok, 1);
3397                 counter_u64_add(rate_limit_active, 1);
3398         } else if (error != EOPNOTSUPP)
3399                   counter_u64_add(rate_limit_alloc_fail, 1);
3400 #endif
3401         return (error);
3402 }
3403
3404 void
3405 in_pcbdetach_tag(struct m_snd_tag *mst)
3406 {
3407
3408         m_snd_tag_rele(mst);
3409 #ifdef INET
3410         counter_u64_add(rate_limit_active, -1);
3411 #endif
3412 }
3413
3414 /*
3415  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3416  * if any:
3417  */
3418 void
3419 in_pcbdetach_txrtlmt(struct inpcb *inp)
3420 {
3421         struct m_snd_tag *mst;
3422
3423         INP_WLOCK_ASSERT(inp);
3424
3425         mst = inp->inp_snd_tag;
3426         inp->inp_snd_tag = NULL;
3427
3428         if (mst == NULL)
3429                 return;
3430
3431         m_snd_tag_rele(mst);
3432 #ifdef INET
3433         counter_u64_add(rate_limit_active, -1);
3434 #endif
3435 }
3436
3437 int
3438 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate)
3439 {
3440         int error;
3441
3442         /*
3443          * If the existing send tag is for the wrong interface due to
3444          * a route change, first drop the existing tag.  Set the
3445          * CHANGED flag so that we will keep trying to allocate a new
3446          * tag if we fail to allocate one this time.
3447          */
3448         if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) {
3449                 in_pcbdetach_txrtlmt(inp);
3450                 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3451         }
3452
3453         /*
3454          * NOTE: When attaching to a network interface a reference is
3455          * made to ensure the network interface doesn't go away until
3456          * all ratelimit connections are gone. The network interface
3457          * pointers compared below represent valid network interfaces,
3458          * except when comparing towards NULL.
3459          */
3460         if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
3461                 error = 0;
3462         } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
3463                 if (inp->inp_snd_tag != NULL)
3464                         in_pcbdetach_txrtlmt(inp);
3465                 error = 0;
3466         } else if (inp->inp_snd_tag == NULL) {
3467                 /*
3468                  * In order to utilize packet pacing with RSS, we need
3469                  * to wait until there is a valid RSS hash before we
3470                  * can proceed:
3471                  */
3472                 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
3473                         error = EAGAIN;
3474                 } else {
3475                         error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3476                             mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag);
3477                 }
3478         } else {
3479                 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3480         }
3481         if (error == 0 || error == EOPNOTSUPP)
3482                 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
3483
3484         return (error);
3485 }
3486
3487 /*
3488  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
3489  * is set in the fast path and will attach/detach/modify the TX rate
3490  * limit send tag based on the socket's so_max_pacing_rate value.
3491  */
3492 void
3493 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
3494 {
3495         struct socket *socket;
3496         uint32_t max_pacing_rate;
3497         bool did_upgrade;
3498
3499         if (inp == NULL)
3500                 return;
3501
3502         socket = inp->inp_socket;
3503         if (socket == NULL)
3504                 return;
3505
3506         if (!INP_WLOCKED(inp)) {
3507                 /*
3508                  * NOTE: If the write locking fails, we need to bail
3509                  * out and use the non-ratelimited ring for the
3510                  * transmit until there is a new chance to get the
3511                  * write lock.
3512                  */
3513                 if (!INP_TRY_UPGRADE(inp))
3514                         return;
3515                 did_upgrade = 1;
3516         } else {
3517                 did_upgrade = 0;
3518         }
3519
3520         /*
3521          * NOTE: The so_max_pacing_rate value is read unlocked,
3522          * because atomic updates are not required since the variable
3523          * is checked at every mbuf we send. It is assumed that the
3524          * variable read itself will be atomic.
3525          */
3526         max_pacing_rate = socket->so_max_pacing_rate;
3527
3528         in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3529
3530         if (did_upgrade)
3531                 INP_DOWNGRADE(inp);
3532 }
3533
3534 /*
3535  * Track route changes for TX rate limiting.
3536  */
3537 void
3538 in_pcboutput_eagain(struct inpcb *inp)
3539 {
3540         bool did_upgrade;
3541
3542         if (inp == NULL)
3543                 return;
3544
3545         if (inp->inp_snd_tag == NULL)
3546                 return;
3547
3548         if (!INP_WLOCKED(inp)) {
3549                 /*
3550                  * NOTE: If the write locking fails, we need to bail
3551                  * out and use the non-ratelimited ring for the
3552                  * transmit until there is a new chance to get the
3553                  * write lock.
3554                  */
3555                 if (!INP_TRY_UPGRADE(inp))
3556                         return;
3557                 did_upgrade = 1;
3558         } else {
3559                 did_upgrade = 0;
3560         }
3561
3562         /* detach rate limiting */
3563         in_pcbdetach_txrtlmt(inp);
3564
3565         /* make sure new mbuf send tag allocation is made */
3566         inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3567
3568         if (did_upgrade)
3569                 INP_DOWNGRADE(inp);
3570 }
3571
3572 #ifdef INET
3573 static void
3574 rl_init(void *st)
3575 {
3576         rate_limit_new = counter_u64_alloc(M_WAITOK);
3577         rate_limit_chg = counter_u64_alloc(M_WAITOK);
3578         rate_limit_active = counter_u64_alloc(M_WAITOK);
3579         rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK);
3580         rate_limit_set_ok = counter_u64_alloc(M_WAITOK);
3581 }
3582
3583 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL);
3584 #endif
3585 #endif /* RATELIMIT */