sys/netinet/in_pcb.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 1982, 1986, 1991, 1993, 1995
   5  *      The Regents of the University of California.
   6  * Copyright (c) 2007-2009 Robert N. M. Watson
   7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
   8  * All rights reserved.
   9  *
  10  * Portions of this software were developed by Robert N. M. Watson under
  11  * contract to Juniper Networks, Inc.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  * 3. Neither the name of the University nor the names of its contributors
  22  *    may be used to endorse or promote products derived from this software
  23  *    without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  *
  37  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  38  */
  39
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include "opt_ddb.h"
  44 #include "opt_ipsec.h"
  45 #include "opt_inet.h"
  46 #include "opt_inet6.h"
  47 #include "opt_ratelimit.h"
  48 #include "opt_pcbgroup.h"
  49 #include "opt_rss.h"
  50
  51 #include <sys/param.h>
  52 #include <sys/systm.h>
  53 #include <sys/lock.h>
  54 #include <sys/malloc.h>
  55 #include <sys/mbuf.h>
  56 #include <sys/callout.h>
  57 #include <sys/eventhandler.h>
  58 #include <sys/domain.h>
  59 #include <sys/protosw.h>
  60 #include <sys/rmlock.h>
  61 #include <sys/socket.h>
  62 #include <sys/socketvar.h>
  63 #include <sys/sockio.h>
  64 #include <sys/priv.h>
  65 #include <sys/proc.h>
  66 #include <sys/refcount.h>
  67 #include <sys/jail.h>
  68 #include <sys/kernel.h>
  69 #include <sys/sysctl.h>
  70
  71 #ifdef DDB
  72 #include <ddb/ddb.h>
  73 #endif
  74
  75 #include <vm/uma.h>
  76
  77 #include <net/if.h>
  78 #include <net/if_var.h>
  79 #include <net/if_types.h>
  80 #include <net/if_llatbl.h>
  81 #include <net/route.h>
  82 #include <net/rss_config.h>
  83 #include <net/vnet.h>
  84
  85 #if defined(INET) || defined(INET6)
  86 #include <netinet/in.h>
  87 #include <netinet/in_pcb.h>
  88 #include <netinet/ip_var.h>
  89 #include <netinet/tcp_var.h>
  90 #include <netinet/udp.h>
  91 #include <netinet/udp_var.h>
  92 #endif
  93 #ifdef INET
  94 #include <netinet/in_var.h>
  95 #endif
  96 #ifdef INET6
  97 #include <netinet/ip6.h>
  98 #include <netinet6/in6_pcb.h>
  99 #include <netinet6/in6_var.h>
 100 #include <netinet6/ip6_var.h>
 101 #endif /* INET6 */
 102
 103 #include <netipsec/ipsec_support.h>
 104
 105 #include <security/mac/mac_framework.h>
 106
 107 static struct callout   ipport_tick_callout;
 108
 109 /*
 110  * These configure the range of local port addresses assigned to
 111  * "unspecified" outgoing connections/packets/whatever.
 112  */
 113 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;    /* 1023 */
 114 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;    /* 600 */
 115 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;     /* 10000 */
 116 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;       /* 65535 */
 117 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;      /* 49152 */
 118 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;        /* 65535 */
 119
 120 /*
 121  * Reserved ports accessible only to root. There are significant
 122  * security considerations that must be accounted for when changing these,
 123  * but the security benefits can be great. Please be careful.
 124  */
 125 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;    /* 1023 */
 126 VNET_DEFINE(int, ipport_reservedlow);
 127
 128 /* Variables dealing with random ephemeral port allocation. */
 129 VNET_DEFINE(int, ipport_randomized) = 1;        /* user controlled via sysctl */
 130 VNET_DEFINE(int, ipport_randomcps) = 10;        /* user controlled via sysctl */
 131 VNET_DEFINE(int, ipport_randomtime) = 45;       /* user controlled via sysctl */
 132 VNET_DEFINE(int, ipport_stoprandom);            /* toggled by ipport_tick */
 133 VNET_DEFINE(int, ipport_tcpallocs);
 134 static VNET_DEFINE(int, ipport_tcplastcount);
 135
 136 #define V_ipport_tcplastcount           VNET(ipport_tcplastcount)
 137
 138 static void     in_pcbremlists(struct inpcb *inp);
 139 #ifdef INET
 140 static struct inpcb     *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
 141                             struct in_addr faddr, u_int fport_arg,
 142                             struct in_addr laddr, u_int lport_arg,
 143                             int lookupflags, struct ifnet *ifp);
 144
 145 #define RANGECHK(var, min, max) \
 146         if ((var) < (min)) { (var) = (min); } \
 147         else if ((var) > (max)) { (var) = (max); }
 148
 149 static int
 150 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 151 {
 152         int error;
 153
 154         error = sysctl_handle_int(oidp, arg1, arg2, req);
 155         if (error == 0) {
 156                 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 157                 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 158                 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
 159                 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
 160                 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
 161                 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
 162         }
 163         return (error);
 164 }
 165
 166 #undef RANGECHK
 167
 168 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0,
 169     "IP Ports");
 170
 171 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
 172         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 173         &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I", "");
 174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
 175         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 176         &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I", "");
 177 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
 178         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 179         &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I", "");
 180 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
 181         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 182         &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I", "");
 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
 184         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 185         &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I", "");
 186 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
 187         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 188         &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I", "");
 189 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
 190         CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
 191         &VNET_NAME(ipport_reservedhigh), 0, "");
 192 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
 193         CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
 194 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
 195         CTLFLAG_VNET | CTLFLAG_RW,
 196         &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
 197 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
 198         CTLFLAG_VNET | CTLFLAG_RW,
 199         &VNET_NAME(ipport_randomcps), 0, "Maximum number of random port "
 200         "allocations before switching to a sequental one");
 201 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
 202         CTLFLAG_VNET | CTLFLAG_RW,
 203         &VNET_NAME(ipport_randomtime), 0,
 204         "Minimum time to keep sequental port "
 205         "allocation before switching to a random one");
 206 #endif /* INET */
 207
 208 /*
 209  * in_pcb.c: manage the Protocol Control Blocks.
 210  *
 211  * NOTE: It is assumed that most of these functions will be called with
 212  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
 213  * functions often modify hash chains or addresses in pcbs.
 214  */
 215
 216 /*
 217  * Different protocols initialize their inpcbs differently - giving
 218  * different name to the lock.  But they all are disposed the same.
 219  */
 220 static void
 221 inpcb_fini(void *mem, int size)
 222 {
 223         struct inpcb *inp = mem;
 224
 225         INP_LOCK_DESTROY(inp);
 226 }
 227
 228 /*
 229  * Initialize an inpcbinfo -- we should be able to reduce the number of
 230  * arguments in time.
 231  */
 232 void
 233 in_pcbinfo_init(struct inpcbinfo *pcbinfo, const char *name,
 234     struct inpcbhead *listhead, int hash_nelements, int porthash_nelements,
 235     char *inpcbzone_name, uma_init inpcbzone_init, u_int hashfields)
 236 {
 237
 238         INP_INFO_LOCK_INIT(pcbinfo, name);
 239         INP_HASH_LOCK_INIT(pcbinfo, "pcbinfohash");     /* XXXRW: argument? */
 240         INP_LIST_LOCK_INIT(pcbinfo, "pcbinfolist");
 241 #ifdef VIMAGE
 242         pcbinfo->ipi_vnet = curvnet;
 243 #endif
 244         pcbinfo->ipi_listhead = listhead;
 245         LIST_INIT(pcbinfo->ipi_listhead);
 246         pcbinfo->ipi_count = 0;
 247         pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB,
 248             &pcbinfo->ipi_hashmask);
 249         pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
 250             &pcbinfo->ipi_porthashmask);
 251 #ifdef PCBGROUP
 252         in_pcbgroup_init(pcbinfo, hashfields, hash_nelements);
 253 #endif
 254         pcbinfo->ipi_zone = uma_zcreate(inpcbzone_name, sizeof(struct inpcb),
 255             NULL, NULL, inpcbzone_init, inpcb_fini, UMA_ALIGN_PTR, 0);
 256         uma_zone_set_max(pcbinfo->ipi_zone, maxsockets);
 257         uma_zone_set_warning(pcbinfo->ipi_zone,
 258             "kern.ipc.maxsockets limit reached");
 259 }
 260
 261 /*
 262  * Destroy an inpcbinfo.
 263  */
 264 void
 265 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
 266 {
 267
 268         KASSERT(pcbinfo->ipi_count == 0,
 269             ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
 270
 271         hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask);
 272         hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
 273             pcbinfo->ipi_porthashmask);
 274 #ifdef PCBGROUP
 275         in_pcbgroup_destroy(pcbinfo);
 276 #endif
 277         uma_zdestroy(pcbinfo->ipi_zone);
 278         INP_LIST_LOCK_DESTROY(pcbinfo);
 279         INP_HASH_LOCK_DESTROY(pcbinfo);
 280         INP_INFO_LOCK_DESTROY(pcbinfo);
 281 }
 282
 283 /*
 284  * Allocate a PCB and associate it with the socket.
 285  * On success return with the PCB locked.
 286  */
 287 int
 288 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 289 {
 290         struct inpcb *inp;
 291         int error;
 292
 293 #ifdef INVARIANTS
 294         if (pcbinfo == &V_tcbinfo) {
 295                 INP_INFO_RLOCK_ASSERT(pcbinfo);
 296         } else {
 297                 INP_INFO_WLOCK_ASSERT(pcbinfo);
 298         }
 299 #endif
 300
 301         error = 0;
 302         inp = uma_zalloc(pcbinfo->ipi_zone, M_NOWAIT);
 303         if (inp == NULL)
 304                 return (ENOBUFS);
 305         bzero(&inp->inp_start_zero, inp_zero_size);
 306         inp->inp_pcbinfo = pcbinfo;
 307         inp->inp_socket = so;
 308         inp->inp_cred = crhold(so->so_cred);
 309         inp->inp_inc.inc_fibnum = so->so_fibnum;
 310 #ifdef MAC
 311         error = mac_inpcb_init(inp, M_NOWAIT);
 312         if (error != 0)
 313                 goto out;
 314         mac_inpcb_create(so, inp);
 315 #endif
 316 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
 317         error = ipsec_init_pcbpolicy(inp);
 318         if (error != 0) {
 319 #ifdef MAC
 320                 mac_inpcb_destroy(inp);
 321 #endif
 322                 goto out;
 323         }
 324 #endif /*IPSEC*/
 325 #ifdef INET6
 326         if (INP_SOCKAF(so) == AF_INET6) {
 327                 inp->inp_vflag |= INP_IPV6PROTO;
 328                 if (V_ip6_v6only)
 329                         inp->inp_flags |= IN6P_IPV6_V6ONLY;
 330         }
 331 #endif
 332         INP_WLOCK(inp);
 333         INP_LIST_WLOCK(pcbinfo);
 334         LIST_INSERT_HEAD(pcbinfo->ipi_listhead, inp, inp_list);
 335         pcbinfo->ipi_count++;
 336         so->so_pcb = (caddr_t)inp;
 337 #ifdef INET6
 338         if (V_ip6_auto_flowlabel)
 339                 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 340 #endif
 341         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 342         refcount_init(&inp->inp_refcount, 1);   /* Reference from inpcbinfo */
 343
 344         /*
 345          * Routes in inpcb's can cache L2 as well; they are guaranteed
 346          * to be cleaned up.
 347          */
 348         inp->inp_route.ro_flags = RT_LLE_CACHE;
 349         INP_LIST_WUNLOCK(pcbinfo);
 350 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
 351 out:
 352         if (error != 0) {
 353                 crfree(inp->inp_cred);
 354                 uma_zfree(pcbinfo->ipi_zone, inp);
 355         }
 356 #endif
 357         return (error);
 358 }
 359
 360 #ifdef INET
 361 int
 362 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 363 {
 364         int anonport, error;
 365
 366         INP_WLOCK_ASSERT(inp);
 367         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 368
 369         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 370                 return (EINVAL);
 371         anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
 372         error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
 373             &inp->inp_lport, cred);
 374         if (error)
 375                 return (error);
 376         if (in_pcbinshash(inp) != 0) {
 377                 inp->inp_laddr.s_addr = INADDR_ANY;
 378                 inp->inp_lport = 0;
 379                 return (EAGAIN);
 380         }
 381         if (anonport)
 382                 inp->inp_flags |= INP_ANONPORT;
 383         return (0);
 384 }
 385 #endif
 386
 387 /*
 388  * Select a local port (number) to use.
 389  */
 390 #if defined(INET) || defined(INET6)
 391 int
 392 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
 393     struct ucred *cred, int lookupflags)
 394 {
 395         struct inpcbinfo *pcbinfo;
 396         struct inpcb *tmpinp;
 397         unsigned short *lastport;
 398         int count, dorandom, error;
 399         u_short aux, first, last, lport;
 400 #ifdef INET
 401         struct in_addr laddr;
 402 #endif
 403
 404         pcbinfo = inp->inp_pcbinfo;
 405
 406         /*
 407          * Because no actual state changes occur here, a global write lock on
 408          * the pcbinfo isn't required.
 409          */
 410         INP_LOCK_ASSERT(inp);
 411         INP_HASH_LOCK_ASSERT(pcbinfo);
 412
 413         if (inp->inp_flags & INP_HIGHPORT) {
 414                 first = V_ipport_hifirstauto;   /* sysctl */
 415                 last  = V_ipport_hilastauto;
 416                 lastport = &pcbinfo->ipi_lasthi;
 417         } else if (inp->inp_flags & INP_LOWPORT) {
 418                 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0);
 419                 if (error)
 420                         return (error);
 421                 first = V_ipport_lowfirstauto;  /* 1023 */
 422                 last  = V_ipport_lowlastauto;   /* 600 */
 423                 lastport = &pcbinfo->ipi_lastlow;
 424         } else {
 425                 first = V_ipport_firstauto;     /* sysctl */
 426                 last  = V_ipport_lastauto;
 427                 lastport = &pcbinfo->ipi_lastport;
 428         }
 429         /*
 430          * For UDP(-Lite), use random port allocation as long as the user
 431          * allows it.  For TCP (and as of yet unknown) connections,
 432          * use random port allocation only if the user allows it AND
 433          * ipport_tick() allows it.
 434          */
 435         if (V_ipport_randomized &&
 436                 (!V_ipport_stoprandom || pcbinfo == &V_udbinfo ||
 437                 pcbinfo == &V_ulitecbinfo))
 438                 dorandom = 1;
 439         else
 440                 dorandom = 0;
 441         /*
 442          * It makes no sense to do random port allocation if
 443          * we have the only port available.
 444          */
 445         if (first == last)
 446                 dorandom = 0;
 447         /* Make sure to not include UDP(-Lite) packets in the count. */
 448         if (pcbinfo != &V_udbinfo || pcbinfo != &V_ulitecbinfo)
 449                 V_ipport_tcpallocs++;
 450         /*
 451          * Instead of having two loops further down counting up or down
 452          * make sure that first is always <= last and go with only one
 453          * code path implementing all logic.
 454          */
 455         if (first > last) {
 456                 aux = first;
 457                 first = last;
 458                 last = aux;
 459         }
 460
 461 #ifdef INET
 462         /* Make the compiler happy. */
 463         laddr.s_addr = 0;
 464         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
 465                 KASSERT(laddrp != NULL, ("%s: laddrp NULL for v4 inp %p",
 466                     __func__, inp));
 467                 laddr = *laddrp;
 468         }
 469 #endif
 470         tmpinp = NULL;  /* Make compiler happy. */
 471         lport = *lportp;
 472
 473         if (dorandom)
 474                 *lastport = first + (arc4random() % (last - first));
 475
 476         count = last - first;
 477
 478         do {
 479                 if (count-- < 0)        /* completely used? */
 480                         return (EADDRNOTAVAIL);
 481                 ++*lastport;
 482                 if (*lastport < first || *lastport > last)
 483                         *lastport = first;
 484                 lport = htons(*lastport);
 485
 486 #ifdef INET6
 487                 if ((inp->inp_vflag & INP_IPV6) != 0)
 488                         tmpinp = in6_pcblookup_local(pcbinfo,
 489                             &inp->in6p_laddr, lport, lookupflags, cred);
 490 #endif
 491 #if defined(INET) && defined(INET6)
 492                 else
 493 #endif
 494 #ifdef INET
 495                         tmpinp = in_pcblookup_local(pcbinfo, laddr,
 496                             lport, lookupflags, cred);
 497 #endif
 498         } while (tmpinp != NULL);
 499
 500 #ifdef INET
 501         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4)
 502                 laddrp->s_addr = laddr.s_addr;
 503 #endif
 504         *lportp = lport;
 505
 506         return (0);
 507 }
 508
 509 /*
 510  * Return cached socket options.
 511  */
 512 short
 513 inp_so_options(const struct inpcb *inp)
 514 {
 515    short so_options;
 516
 517    so_options = 0;
 518
 519    if ((inp->inp_flags2 & INP_REUSEPORT) != 0)
 520            so_options |= SO_REUSEPORT;
 521    if ((inp->inp_flags2 & INP_REUSEADDR) != 0)
 522            so_options |= SO_REUSEADDR;
 523    return (so_options);
 524 }
 525 #endif /* INET || INET6 */
 526
 527 /*
 528  * Check if a new BINDMULTI socket is allowed to be created.
 529  *
 530  * ni points to the new inp.
 531  * oi points to the exisitng inp.
 532  *
 533  * This checks whether the existing inp also has BINDMULTI and
 534  * whether the credentials match.
 535  */
 536 int
 537 in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
 538 {
 539         /* Check permissions match */
 540         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 541             (ni->inp_cred->cr_uid !=
 542             oi->inp_cred->cr_uid))
 543                 return (0);
 544
 545         /* Check the existing inp has BINDMULTI set */
 546         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 547             ((oi->inp_flags2 & INP_BINDMULTI) == 0))
 548                 return (0);
 549
 550         /*
 551          * We're okay - either INP_BINDMULTI isn't set on ni, or
 552          * it is and it matches the checks.
 553          */
 554         return (1);
 555 }
 556
 557 #ifdef INET
 558 /*
 559  * Set up a bind operation on a PCB, performing port allocation
 560  * as required, but do not actually modify the PCB. Callers can
 561  * either complete the bind by setting inp_laddr/inp_lport and
 562  * calling in_pcbinshash(), or they can just use the resulting
 563  * port and address to authorise the sending of a once-off packet.
 564  *
 565  * On error, the values of *laddrp and *lportp are not changed.
 566  */
 567 int
 568 in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
 569     u_short *lportp, struct ucred *cred)
 570 {
 571         struct socket *so = inp->inp_socket;
 572         struct sockaddr_in *sin;
 573         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 574         struct in_addr laddr;
 575         u_short lport = 0;
 576         int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
 577         int error;
 578
 579         /*
 580          * No state changes, so read locks are sufficient here.
 581          */
 582         INP_LOCK_ASSERT(inp);
 583         INP_HASH_LOCK_ASSERT(pcbinfo);
 584
 585         if (TAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */
 586                 return (EADDRNOTAVAIL);
 587         laddr.s_addr = *laddrp;
 588         if (nam != NULL && laddr.s_addr != INADDR_ANY)
 589                 return (EINVAL);
 590         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) == 0)
 591                 lookupflags = INPLOOKUP_WILDCARD;
 592         if (nam == NULL) {
 593                 if ((error = prison_local_ip4(cred, &laddr)) != 0)
 594                         return (error);
 595         } else {
 596                 sin = (struct sockaddr_in *)nam;
 597                 if (nam->sa_len != sizeof (*sin))
 598                         return (EINVAL);
 599 #ifdef notdef
 600                 /*
 601                  * We should check the family, but old programs
 602                  * incorrectly fail to initialize it.
 603                  */
 604                 if (sin->sin_family != AF_INET)
 605                         return (EAFNOSUPPORT);
 606 #endif
 607                 error = prison_local_ip4(cred, &sin->sin_addr);
 608                 if (error)
 609                         return (error);
 610                 if (sin->sin_port != *lportp) {
 611                         /* Don't allow the port to change. */
 612                         if (*lportp != 0)
 613                                 return (EINVAL);
 614                         lport = sin->sin_port;
 615                 }
 616                 /* NB: lport is left as 0 if the port isn't being changed. */
 617                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 618                         /*
 619                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 620                          * allow complete duplication of binding if
 621                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 622                          * and a multicast address is bound on both
 623                          * new and duplicated sockets.
 624                          */
 625                         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
 626                                 reuseport = SO_REUSEADDR|SO_REUSEPORT;
 627                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 628                         sin->sin_port = 0;              /* yech... */
 629                         bzero(&sin->sin_zero, sizeof(sin->sin_zero));
 630                         /*
 631                          * Is the address a local IP address?
 632                          * If INP_BINDANY is set, then the socket may be bound
 633                          * to any endpoint address, local or not.
 634                          */
 635                         if ((inp->inp_flags & INP_BINDANY) == 0 &&
 636                             ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
 637                                 return (EADDRNOTAVAIL);
 638                 }
 639                 laddr = sin->sin_addr;
 640                 if (lport) {
 641                         struct inpcb *t;
 642                         struct tcptw *tw;
 643
 644                         /* GROSS */
 645                         if (ntohs(lport) <= V_ipport_reservedhigh &&
 646                             ntohs(lport) >= V_ipport_reservedlow &&
 647                             priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT,
 648                             0))
 649                                 return (EACCES);
 650                         if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 651                             priv_check_cred(inp->inp_cred,
 652                             PRIV_NETINET_REUSEPORT, 0) != 0) {
 653                                 t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 654                                     lport, INPLOOKUP_WILDCARD, cred);
 655         /*
 656          * XXX
 657          * This entire block sorely needs a rewrite.
 658          */
 659                                 if (t &&
 660                                     ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 661                                     ((t->inp_flags & INP_TIMEWAIT) == 0) &&
 662                                     (so->so_type != SOCK_STREAM ||
 663                                      ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
 664                                     (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
 665                                      ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
 666                                      (t->inp_flags2 & INP_REUSEPORT) == 0) &&
 667                                     (inp->inp_cred->cr_uid !=
 668                                      t->inp_cred->cr_uid))
 669                                         return (EADDRINUSE);
 670
 671                                 /*
 672                                  * If the socket is a BINDMULTI socket, then
 673                                  * the credentials need to match and the
 674                                  * original socket also has to have been bound
 675                                  * with BINDMULTI.
 676                                  */
 677                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 678                                         return (EADDRINUSE);
 679                         }
 680                         t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 681                             lport, lookupflags, cred);
 682                         if (t && (t->inp_flags & INP_TIMEWAIT)) {
 683                                 /*
 684                                  * XXXRW: If an incpb has had its timewait
 685                                  * state recycled, we treat the address as
 686                                  * being in use (for now).  This is better
 687                                  * than a panic, but not desirable.
 688                                  */
 689                                 tw = intotw(t);
 690                                 if (tw == NULL ||
 691                                     (reuseport & tw->tw_so_options) == 0)
 692                                         return (EADDRINUSE);
 693                         } else if (t &&
 694                             ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 695                             (reuseport & inp_so_options(t)) == 0) {
 696 #ifdef INET6
 697                                 if (ntohl(sin->sin_addr.s_addr) !=
 698                                     INADDR_ANY ||
 699                                     ntohl(t->inp_laddr.s_addr) !=
 700                                     INADDR_ANY ||
 701                                     (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
 702                                     (t->inp_vflag & INP_IPV6PROTO) == 0)
 703 #endif
 704                                 return (EADDRINUSE);
 705                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 706                                         return (EADDRINUSE);
 707                         }
 708                 }
 709         }
 710         if (*lportp != 0)
 711                 lport = *lportp;
 712         if (lport == 0) {
 713                 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
 714                 if (error != 0)
 715                         return (error);
 716
 717         }
 718         *laddrp = laddr.s_addr;
 719         *lportp = lport;
 720         return (0);
 721 }
 722
 723 /*
 724  * Connect from a socket to a specified address.
 725  * Both address and port must be specified in argument sin.
 726  * If don't have a local address for this socket yet,
 727  * then pick one.
 728  */
 729 int
 730 in_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
 731     struct ucred *cred, struct mbuf *m)
 732 {
 733         u_short lport, fport;
 734         in_addr_t laddr, faddr;
 735         int anonport, error;
 736
 737         INP_WLOCK_ASSERT(inp);
 738         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 739
 740         lport = inp->inp_lport;
 741         laddr = inp->inp_laddr.s_addr;
 742         anonport = (lport == 0);
 743         error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport,
 744             NULL, cred);
 745         if (error)
 746                 return (error);
 747
 748         /* Do the initial binding of the local address if required. */
 749         if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
 750                 inp->inp_lport = lport;
 751                 inp->inp_laddr.s_addr = laddr;
 752                 if (in_pcbinshash(inp) != 0) {
 753                         inp->inp_laddr.s_addr = INADDR_ANY;
 754                         inp->inp_lport = 0;
 755                         return (EAGAIN);
 756                 }
 757         }
 758
 759         /* Commit the remaining changes. */
 760         inp->inp_lport = lport;
 761         inp->inp_laddr.s_addr = laddr;
 762         inp->inp_faddr.s_addr = faddr;
 763         inp->inp_fport = fport;
 764         in_pcbrehash_mbuf(inp, m);
 765
 766         if (anonport)
 767                 inp->inp_flags |= INP_ANONPORT;
 768         return (0);
 769 }
 770
 771 int
 772 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 773 {
 774
 775         return (in_pcbconnect_mbuf(inp, nam, cred, NULL));
 776 }
 777
 778 /*
 779  * Do proper source address selection on an unbound socket in case
 780  * of connect. Take jails into account as well.
 781  */
 782 int
 783 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
 784     struct ucred *cred)
 785 {
 786         struct ifaddr *ifa;
 787         struct sockaddr *sa;
 788         struct sockaddr_in *sin;
 789         struct route sro;
 790         int error;
 791
 792         KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
 793
 794         /*
 795          * Bypass source address selection and use the primary jail IP
 796          * if requested.
 797          */
 798         if (cred != NULL && !prison_saddrsel_ip4(cred, laddr))
 799                 return (0);
 800
 801         error = 0;
 802         bzero(&sro, sizeof(sro));
 803
 804         sin = (struct sockaddr_in *)&sro.ro_dst;
 805         sin->sin_family = AF_INET;
 806         sin->sin_len = sizeof(struct sockaddr_in);
 807         sin->sin_addr.s_addr = faddr->s_addr;
 808
 809         /*
 810          * If route is known our src addr is taken from the i/f,
 811          * else punt.
 812          *
 813          * Find out route to destination.
 814          */
 815         if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
 816                 in_rtalloc_ign(&sro, 0, inp->inp_inc.inc_fibnum);
 817
 818         /*
 819          * If we found a route, use the address corresponding to
 820          * the outgoing interface.
 821          *
 822          * Otherwise assume faddr is reachable on a directly connected
 823          * network and try to find a corresponding interface to take
 824          * the source address from.
 825          */
 826         if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) {
 827                 struct in_ifaddr *ia;
 828                 struct ifnet *ifp;
 829
 830                 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
 831                                         inp->inp_socket->so_fibnum));
 832                 if (ia == NULL)
 833                         ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
 834                                                 inp->inp_socket->so_fibnum));
 835                 if (ia == NULL) {
 836                         error = ENETUNREACH;
 837                         goto done;
 838                 }
 839
 840                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 841                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 842                         ifa_free(&ia->ia_ifa);
 843                         goto done;
 844                 }
 845
 846                 ifp = ia->ia_ifp;
 847                 ifa_free(&ia->ia_ifa);
 848                 ia = NULL;
 849                 IF_ADDR_RLOCK(ifp);
 850                 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 851
 852                         sa = ifa->ifa_addr;
 853                         if (sa->sa_family != AF_INET)
 854                                 continue;
 855                         sin = (struct sockaddr_in *)sa;
 856                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 857                                 ia = (struct in_ifaddr *)ifa;
 858                                 break;
 859                         }
 860                 }
 861                 if (ia != NULL) {
 862                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 863                         IF_ADDR_RUNLOCK(ifp);
 864                         goto done;
 865                 }
 866                 IF_ADDR_RUNLOCK(ifp);
 867
 868                 /* 3. As a last resort return the 'default' jail address. */
 869                 error = prison_get_ip4(cred, laddr);
 870                 goto done;
 871         }
 872
 873         /*
 874          * If the outgoing interface on the route found is not
 875          * a loopback interface, use the address from that interface.
 876          * In case of jails do those three steps:
 877          * 1. check if the interface address belongs to the jail. If so use it.
 878          * 2. check if we have any address on the outgoing interface
 879          *    belonging to this jail. If so use it.
 880          * 3. as a last resort return the 'default' jail address.
 881          */
 882         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) == 0) {
 883                 struct in_ifaddr *ia;
 884                 struct ifnet *ifp;
 885
 886                 /* If not jailed, use the default returned. */
 887                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 888                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 889                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 890                         goto done;
 891                 }
 892
 893                 /* Jailed. */
 894                 /* 1. Check if the iface address belongs to the jail. */
 895                 sin = (struct sockaddr_in *)sro.ro_rt->rt_ifa->ifa_addr;
 896                 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 897                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 898                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 899                         goto done;
 900                 }
 901
 902                 /*
 903                  * 2. Check if we have any address on the outgoing interface
 904                  *    belonging to this jail.
 905                  */
 906                 ia = NULL;
 907                 ifp = sro.ro_rt->rt_ifp;
 908                 IF_ADDR_RLOCK(ifp);
 909                 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 910                         sa = ifa->ifa_addr;
 911                         if (sa->sa_family != AF_INET)
 912                                 continue;
 913                         sin = (struct sockaddr_in *)sa;
 914                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 915                                 ia = (struct in_ifaddr *)ifa;
 916                                 break;
 917                         }
 918                 }
 919                 if (ia != NULL) {
 920                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 921                         IF_ADDR_RUNLOCK(ifp);
 922                         goto done;
 923                 }
 924                 IF_ADDR_RUNLOCK(ifp);
 925
 926                 /* 3. As a last resort return the 'default' jail address. */
 927                 error = prison_get_ip4(cred, laddr);
 928                 goto done;
 929         }
 930
 931         /*
 932          * The outgoing interface is marked with 'loopback net', so a route
 933          * to ourselves is here.
 934          * Try to find the interface of the destination address and then
 935          * take the address from there. That interface is not necessarily
 936          * a loopback interface.
 937          * In case of jails, check that it is an address of the jail
 938          * and if we cannot find, fall back to the 'default' jail address.
 939          */
 940         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
 941                 struct sockaddr_in sain;
 942                 struct in_ifaddr *ia;
 943
 944                 bzero(&sain, sizeof(struct sockaddr_in));
 945                 sain.sin_family = AF_INET;
 946                 sain.sin_len = sizeof(struct sockaddr_in);
 947                 sain.sin_addr.s_addr = faddr->s_addr;
 948
 949                 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&sain),
 950                                         inp->inp_socket->so_fibnum));
 951                 if (ia == NULL)
 952                         ia = ifatoia(ifa_ifwithnet(sintosa(&sain), 0,
 953                                                 inp->inp_socket->so_fibnum));
 954                 if (ia == NULL)
 955                         ia = ifatoia(ifa_ifwithaddr(sintosa(&sain)));
 956
 957                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 958                         if (ia == NULL) {
 959                                 error = ENETUNREACH;
 960                                 goto done;
 961                         }
 962                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 963                         ifa_free(&ia->ia_ifa);
 964                         goto done;
 965                 }
 966
 967                 /* Jailed. */
 968                 if (ia != NULL) {
 969                         struct ifnet *ifp;
 970
 971                         ifp = ia->ia_ifp;
 972                         ifa_free(&ia->ia_ifa);
 973                         ia = NULL;
 974                         IF_ADDR_RLOCK(ifp);
 975                         TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 976
 977                                 sa = ifa->ifa_addr;
 978                                 if (sa->sa_family != AF_INET)
 979                                         continue;
 980                                 sin = (struct sockaddr_in *)sa;
 981                                 if (prison_check_ip4(cred,
 982                                     &sin->sin_addr) == 0) {
 983                                         ia = (struct in_ifaddr *)ifa;
 984                                         break;
 985                                 }
 986                         }
 987                         if (ia != NULL) {
 988                                 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 989                                 IF_ADDR_RUNLOCK(ifp);
 990                                 goto done;
 991                         }
 992                         IF_ADDR_RUNLOCK(ifp);
 993                 }
 994
 995                 /* 3. As a last resort return the 'default' jail address. */
 996                 error = prison_get_ip4(cred, laddr);
 997                 goto done;
 998         }
 999
1000 done:
1001         if (sro.ro_rt != NULL)
1002                 RTFREE(sro.ro_rt);
1003         return (error);
1004 }
1005
1006 /*
1007  * Set up for a connect from a socket to the specified address.
1008  * On entry, *laddrp and *lportp should contain the current local
1009  * address and port for the PCB; these are updated to the values
1010  * that should be placed in inp_laddr and inp_lport to complete
1011  * the connect.
1012  *
1013  * On success, *faddrp and *fportp will be set to the remote address
1014  * and port. These are not updated in the error case.
1015  *
1016  * If the operation fails because the connection already exists,
1017  * *oinpp will be set to the PCB of that connection so that the
1018  * caller can decide to override it. In all other cases, *oinpp
1019  * is set to NULL.
1020  */
1021 int
1022 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
1023     in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1024     struct inpcb **oinpp, struct ucred *cred)
1025 {
1026         struct rm_priotracker in_ifa_tracker;
1027         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1028         struct in_ifaddr *ia;
1029         struct inpcb *oinp;
1030         struct in_addr laddr, faddr;
1031         u_short lport, fport;
1032         int error;
1033
1034         /*
1035          * Because a global state change doesn't actually occur here, a read
1036          * lock is sufficient.
1037          */
1038         INP_LOCK_ASSERT(inp);
1039         INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1040
1041         if (oinpp != NULL)
1042                 *oinpp = NULL;
1043         if (nam->sa_len != sizeof (*sin))
1044                 return (EINVAL);
1045         if (sin->sin_family != AF_INET)
1046                 return (EAFNOSUPPORT);
1047         if (sin->sin_port == 0)
1048                 return (EADDRNOTAVAIL);
1049         laddr.s_addr = *laddrp;
1050         lport = *lportp;
1051         faddr = sin->sin_addr;
1052         fport = sin->sin_port;
1053
1054         if (!TAILQ_EMPTY(&V_in_ifaddrhead)) {
1055                 /*
1056                  * If the destination address is INADDR_ANY,
1057                  * use the primary local address.
1058                  * If the supplied address is INADDR_BROADCAST,
1059                  * and the primary interface supports broadcast,
1060                  * choose the broadcast address for that interface.
1061                  */
1062                 if (faddr.s_addr == INADDR_ANY) {
1063                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1064                         faddr =
1065                             IA_SIN(TAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1066                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1067                         if (cred != NULL &&
1068                             (error = prison_get_ip4(cred, &faddr)) != 0)
1069                                 return (error);
1070                 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1071                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1072                         if (TAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1073                             IFF_BROADCAST)
1074                                 faddr = satosin(&TAILQ_FIRST(
1075                                     &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1076                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1077                 }
1078         }
1079         if (laddr.s_addr == INADDR_ANY) {
1080                 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1081                 /*
1082                  * If the destination address is multicast and an outgoing
1083                  * interface has been set as a multicast option, prefer the
1084                  * address of that interface as our source address.
1085                  */
1086                 if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1087                     inp->inp_moptions != NULL) {
1088                         struct ip_moptions *imo;
1089                         struct ifnet *ifp;
1090
1091                         imo = inp->inp_moptions;
1092                         if (imo->imo_multicast_ifp != NULL) {
1093                                 ifp = imo->imo_multicast_ifp;
1094                                 IN_IFADDR_RLOCK(&in_ifa_tracker);
1095                                 TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1096                                         if ((ia->ia_ifp == ifp) &&
1097                                             (cred == NULL ||
1098                                             prison_check_ip4(cred,
1099                                             &ia->ia_addr.sin_addr) == 0))
1100                                                 break;
1101                                 }
1102                                 if (ia == NULL)
1103                                         error = EADDRNOTAVAIL;
1104                                 else {
1105                                         laddr = ia->ia_addr.sin_addr;
1106                                         error = 0;
1107                                 }
1108                                 IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1109                         }
1110                 }
1111                 if (error)
1112                         return (error);
1113         }
1114         oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, fport,
1115             laddr, lport, 0, NULL);
1116         if (oinp != NULL) {
1117                 if (oinpp != NULL)
1118                         *oinpp = oinp;
1119                 return (EADDRINUSE);
1120         }
1121         if (lport == 0) {
1122                 error = in_pcbbind_setup(inp, NULL, &laddr.s_addr, &lport,
1123                     cred);
1124                 if (error)
1125                         return (error);
1126         }
1127         *laddrp = laddr.s_addr;
1128         *lportp = lport;
1129         *faddrp = faddr.s_addr;
1130         *fportp = fport;
1131         return (0);
1132 }
1133
1134 void
1135 in_pcbdisconnect(struct inpcb *inp)
1136 {
1137
1138         INP_WLOCK_ASSERT(inp);
1139         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1140
1141         inp->inp_faddr.s_addr = INADDR_ANY;
1142         inp->inp_fport = 0;
1143         in_pcbrehash(inp);
1144 }
1145 #endif /* INET */
1146
1147 /*
1148  * in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
1149  * For most protocols, this will be invoked immediately prior to calling
1150  * in_pcbfree().  However, with TCP the inpcb may significantly outlive the
1151  * socket, in which case in_pcbfree() is deferred.
1152  */
1153 void
1154 in_pcbdetach(struct inpcb *inp)
1155 {
1156
1157         KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1158
1159 #ifdef RATELIMIT
1160         if (inp->inp_snd_tag != NULL)
1161                 in_pcbdetach_txrtlmt(inp);
1162 #endif
1163         inp->inp_socket->so_pcb = NULL;
1164         inp->inp_socket = NULL;
1165 }
1166
1167 /*
1168  * in_pcbref() bumps the reference count on an inpcb in order to maintain
1169  * stability of an inpcb pointer despite the inpcb lock being released.  This
1170  * is used in TCP when the inpcbinfo lock needs to be acquired or upgraded,
1171  * but where the inpcb lock may already held, or when acquiring a reference
1172  * via a pcbgroup.
1173  *
1174  * in_pcbref() should be used only to provide brief memory stability, and
1175  * must always be followed by a call to INP_WLOCK() and in_pcbrele() to
1176  * garbage collect the inpcb if it has been in_pcbfree()'d from another
1177  * context.  Until in_pcbrele() has returned that the inpcb is still valid,
1178  * lock and rele are the *only* safe operations that may be performed on the
1179  * inpcb.
1180  *
1181  * While the inpcb will not be freed, releasing the inpcb lock means that the
1182  * connection's state may change, so the caller should be careful to
1183  * revalidate any cached state on reacquiring the lock.  Drop the reference
1184  * using in_pcbrele().
1185  */
1186 void
1187 in_pcbref(struct inpcb *inp)
1188 {
1189
1190         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1191
1192         refcount_acquire(&inp->inp_refcount);
1193 }
1194
1195 /*
1196  * Drop a refcount on an inpcb elevated using in_pcbref(); because a call to
1197  * in_pcbfree() may have been made between in_pcbref() and in_pcbrele(), we
1198  * return a flag indicating whether or not the inpcb remains valid.  If it is
1199  * valid, we return with the inpcb lock held.
1200  *
1201  * Notice that, unlike in_pcbref(), the inpcb lock must be held to drop a
1202  * reference on an inpcb.  Historically more work was done here (actually, in
1203  * in_pcbfree_internal()) but has been moved to in_pcbfree() to avoid the
1204  * need for the pcbinfo lock in in_pcbrele().  Deferring the free is entirely
1205  * about memory stability (and continued use of the write lock).
1206  */
1207 int
1208 in_pcbrele_rlocked(struct inpcb *inp)
1209 {
1210         struct inpcbinfo *pcbinfo;
1211
1212         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1213
1214         INP_RLOCK_ASSERT(inp);
1215
1216         if (refcount_release(&inp->inp_refcount) == 0) {
1217                 /*
1218                  * If the inpcb has been freed, let the caller know, even if
1219                  * this isn't the last reference.
1220                  */
1221                 if (inp->inp_flags2 & INP_FREED) {
1222                         INP_RUNLOCK(inp);
1223                         return (1);
1224                 }
1225                 return (0);
1226         }
1227
1228         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1229
1230         INP_RUNLOCK(inp);
1231         pcbinfo = inp->inp_pcbinfo;
1232         uma_zfree(pcbinfo->ipi_zone, inp);
1233         return (1);
1234 }
1235
1236 int
1237 in_pcbrele_wlocked(struct inpcb *inp)
1238 {
1239         struct inpcbinfo *pcbinfo;
1240
1241         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1242
1243         INP_WLOCK_ASSERT(inp);
1244
1245         if (refcount_release(&inp->inp_refcount) == 0) {
1246                 /*
1247                  * If the inpcb has been freed, let the caller know, even if
1248                  * this isn't the last reference.
1249                  */
1250                 if (inp->inp_flags2 & INP_FREED) {
1251                         INP_WUNLOCK(inp);
1252                         return (1);
1253                 }
1254                 return (0);
1255         }
1256
1257         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1258
1259         INP_WUNLOCK(inp);
1260         pcbinfo = inp->inp_pcbinfo;
1261         uma_zfree(pcbinfo->ipi_zone, inp);
1262         return (1);
1263 }
1264
1265 /*
1266  * Temporary wrapper.
1267  */
1268 int
1269 in_pcbrele(struct inpcb *inp)
1270 {
1271
1272         return (in_pcbrele_wlocked(inp));
1273 }
1274
1275 /*
1276  * Unconditionally schedule an inpcb to be freed by decrementing its
1277  * reference count, which should occur only after the inpcb has been detached
1278  * from its socket.  If another thread holds a temporary reference (acquired
1279  * using in_pcbref()) then the free is deferred until that reference is
1280  * released using in_pcbrele(), but the inpcb is still unlocked.  Almost all
1281  * work, including removal from global lists, is done in this context, where
1282  * the pcbinfo lock is held.
1283  */
1284 void
1285 in_pcbfree(struct inpcb *inp)
1286 {
1287         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1288
1289         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1290
1291 #ifdef INVARIANTS
1292         if (pcbinfo == &V_tcbinfo) {
1293                 INP_INFO_LOCK_ASSERT(pcbinfo);
1294         } else {
1295                 INP_INFO_WLOCK_ASSERT(pcbinfo);
1296         }
1297 #endif
1298         INP_WLOCK_ASSERT(inp);
1299
1300         /* XXXRW: Do as much as possible here. */
1301 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1302         if (inp->inp_sp != NULL)
1303                 ipsec_delete_pcbpolicy(inp);
1304 #endif
1305         INP_LIST_WLOCK(pcbinfo);
1306         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1307         in_pcbremlists(inp);
1308         INP_LIST_WUNLOCK(pcbinfo);
1309 #ifdef INET6
1310         if (inp->inp_vflag & INP_IPV6PROTO) {
1311                 ip6_freepcbopts(inp->in6p_outputopts);
1312                 if (inp->in6p_moptions != NULL)
1313                         ip6_freemoptions(inp->in6p_moptions);
1314         }
1315 #endif
1316         if (inp->inp_options)
1317                 (void)m_free(inp->inp_options);
1318 #ifdef INET
1319         if (inp->inp_moptions != NULL)
1320                 inp_freemoptions(inp->inp_moptions);
1321 #endif
1322         RO_INVALIDATE_CACHE(&inp->inp_route);
1323
1324         inp->inp_vflag = 0;
1325         inp->inp_flags2 |= INP_FREED;
1326         crfree(inp->inp_cred);
1327 #ifdef MAC
1328         mac_inpcb_destroy(inp);
1329 #endif
1330         if (!in_pcbrele_wlocked(inp))
1331                 INP_WUNLOCK(inp);
1332 }
1333
1334 /*
1335  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1336  * port reservation, and preventing it from being returned by inpcb lookups.
1337  *
1338  * It is used by TCP to mark an inpcb as unused and avoid future packet
1339  * delivery or event notification when a socket remains open but TCP has
1340  * closed.  This might occur as a result of a shutdown()-initiated TCP close
1341  * or a RST on the wire, and allows the port binding to be reused while still
1342  * maintaining the invariant that so_pcb always points to a valid inpcb until
1343  * in_pcbdetach().
1344  *
1345  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1346  * in_pcbnotifyall() and in_pcbpurgeif0()?
1347  */
1348 void
1349 in_pcbdrop(struct inpcb *inp)
1350 {
1351
1352         INP_WLOCK_ASSERT(inp);
1353
1354         /*
1355          * XXXRW: Possibly we should protect the setting of INP_DROPPED with
1356          * the hash lock...?
1357          */
1358         inp->inp_flags |= INP_DROPPED;
1359         if (inp->inp_flags & INP_INHASHLIST) {
1360                 struct inpcbport *phd = inp->inp_phd;
1361
1362                 INP_HASH_WLOCK(inp->inp_pcbinfo);
1363                 LIST_REMOVE(inp, inp_hash);
1364                 LIST_REMOVE(inp, inp_portlist);
1365                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
1366                         LIST_REMOVE(phd, phd_hash);
1367                         free(phd, M_PCB);
1368                 }
1369                 INP_HASH_WUNLOCK(inp->inp_pcbinfo);
1370                 inp->inp_flags &= ~INP_INHASHLIST;
1371 #ifdef PCBGROUP
1372                 in_pcbgroup_remove(inp);
1373 #endif
1374         }
1375 }
1376
1377 #ifdef INET
1378 /*
1379  * Common routines to return the socket addresses associated with inpcbs.
1380  */
1381 struct sockaddr *
1382 in_sockaddr(in_port_t port, struct in_addr *addr_p)
1383 {
1384         struct sockaddr_in *sin;
1385
1386         sin = malloc(sizeof *sin, M_SONAME,
1387                 M_WAITOK | M_ZERO);
1388         sin->sin_family = AF_INET;
1389         sin->sin_len = sizeof(*sin);
1390         sin->sin_addr = *addr_p;
1391         sin->sin_port = port;
1392
1393         return (struct sockaddr *)sin;
1394 }
1395
1396 int
1397 in_getsockaddr(struct socket *so, struct sockaddr **nam)
1398 {
1399         struct inpcb *inp;
1400         struct in_addr addr;
1401         in_port_t port;
1402
1403         inp = sotoinpcb(so);
1404         KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1405
1406         INP_RLOCK(inp);
1407         port = inp->inp_lport;
1408         addr = inp->inp_laddr;
1409         INP_RUNLOCK(inp);
1410
1411         *nam = in_sockaddr(port, &addr);
1412         return 0;
1413 }
1414
1415 int
1416 in_getpeeraddr(struct socket *so, struct sockaddr **nam)
1417 {
1418         struct inpcb *inp;
1419         struct in_addr addr;
1420         in_port_t port;
1421
1422         inp = sotoinpcb(so);
1423         KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1424
1425         INP_RLOCK(inp);
1426         port = inp->inp_fport;
1427         addr = inp->inp_faddr;
1428         INP_RUNLOCK(inp);
1429
1430         *nam = in_sockaddr(port, &addr);
1431         return 0;
1432 }
1433
1434 void
1435 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
1436     struct inpcb *(*notify)(struct inpcb *, int))
1437 {
1438         struct inpcb *inp, *inp_temp;
1439
1440         INP_INFO_WLOCK(pcbinfo);
1441         LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
1442                 INP_WLOCK(inp);
1443 #ifdef INET6
1444                 if ((inp->inp_vflag & INP_IPV4) == 0) {
1445                         INP_WUNLOCK(inp);
1446                         continue;
1447                 }
1448 #endif
1449                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1450                     inp->inp_socket == NULL) {
1451                         INP_WUNLOCK(inp);
1452                         continue;
1453                 }
1454                 if ((*notify)(inp, errno))
1455                         INP_WUNLOCK(inp);
1456         }
1457         INP_INFO_WUNLOCK(pcbinfo);
1458 }
1459
1460 void
1461 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1462 {
1463         struct inpcb *inp;
1464         struct ip_moptions *imo;
1465         int i, gap;
1466
1467         INP_INFO_WLOCK(pcbinfo);
1468         LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1469                 INP_WLOCK(inp);
1470                 imo = inp->inp_moptions;
1471                 if ((inp->inp_vflag & INP_IPV4) &&
1472                     imo != NULL) {
1473                         /*
1474                          * Unselect the outgoing interface if it is being
1475                          * detached.
1476                          */
1477                         if (imo->imo_multicast_ifp == ifp)
1478                                 imo->imo_multicast_ifp = NULL;
1479
1480                         /*
1481                          * Drop multicast group membership if we joined
1482                          * through the interface being detached.
1483                          */
1484                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1485                             i++) {
1486                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1487                                         in_delmulti(imo->imo_membership[i]);
1488                                         gap++;
1489                                 } else if (gap != 0)
1490                                         imo->imo_membership[i - gap] =
1491                                             imo->imo_membership[i];
1492                         }
1493                         imo->imo_num_memberships -= gap;
1494                 }
1495                 INP_WUNLOCK(inp);
1496         }
1497         INP_INFO_WUNLOCK(pcbinfo);
1498 }
1499
1500 /*
1501  * Lookup a PCB based on the local address and port.  Caller must hold the
1502  * hash lock.  No inpcb locks or references are acquired.
1503  */
1504 #define INP_LOOKUP_MAPPED_PCB_COST      3
1505 struct inpcb *
1506 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1507     u_short lport, int lookupflags, struct ucred *cred)
1508 {
1509         struct inpcb *inp;
1510 #ifdef INET6
1511         int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1512 #else
1513         int matchwild = 3;
1514 #endif
1515         int wildcard;
1516
1517         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1518             ("%s: invalid lookup flags %d", __func__, lookupflags));
1519
1520         INP_HASH_LOCK_ASSERT(pcbinfo);
1521
1522         if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1523                 struct inpcbhead *head;
1524                 /*
1525                  * Look for an unconnected (wildcard foreign addr) PCB that
1526                  * matches the local address and port we're looking for.
1527                  */
1528                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1529                     0, pcbinfo->ipi_hashmask)];
1530                 LIST_FOREACH(inp, head, inp_hash) {
1531 #ifdef INET6
1532                         /* XXX inp locking */
1533                         if ((inp->inp_vflag & INP_IPV4) == 0)
1534                                 continue;
1535 #endif
1536                         if (inp->inp_faddr.s_addr == INADDR_ANY &&
1537                             inp->inp_laddr.s_addr == laddr.s_addr &&
1538                             inp->inp_lport == lport) {
1539                                 /*
1540                                  * Found?
1541                                  */
1542                                 if (cred == NULL ||
1543                                     prison_equal_ip4(cred->cr_prison,
1544                                         inp->inp_cred->cr_prison))
1545                                         return (inp);
1546                         }
1547                 }
1548                 /*
1549                  * Not found.
1550                  */
1551                 return (NULL);
1552         } else {
1553                 struct inpcbporthead *porthash;
1554                 struct inpcbport *phd;
1555                 struct inpcb *match = NULL;
1556                 /*
1557                  * Best fit PCB lookup.
1558                  *
1559                  * First see if this local port is in use by looking on the
1560                  * port hash list.
1561                  */
1562                 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
1563                     pcbinfo->ipi_porthashmask)];
1564                 LIST_FOREACH(phd, porthash, phd_hash) {
1565                         if (phd->phd_port == lport)
1566                                 break;
1567                 }
1568                 if (phd != NULL) {
1569                         /*
1570                          * Port is in use by one or more PCBs. Look for best
1571                          * fit.
1572                          */
1573                         LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1574                                 wildcard = 0;
1575                                 if (cred != NULL &&
1576                                     !prison_equal_ip4(inp->inp_cred->cr_prison,
1577                                         cred->cr_prison))
1578                                         continue;
1579 #ifdef INET6
1580                                 /* XXX inp locking */
1581                                 if ((inp->inp_vflag & INP_IPV4) == 0)
1582                                         continue;
1583                                 /*
1584                                  * We never select the PCB that has
1585                                  * INP_IPV6 flag and is bound to :: if
1586                                  * we have another PCB which is bound
1587                                  * to 0.0.0.0.  If a PCB has the
1588                                  * INP_IPV6 flag, then we set its cost
1589                                  * higher than IPv4 only PCBs.
1590                                  *
1591                                  * Note that the case only happens
1592                                  * when a socket is bound to ::, under
1593                                  * the condition that the use of the
1594                                  * mapped address is allowed.
1595                                  */
1596                                 if ((inp->inp_vflag & INP_IPV6) != 0)
1597                                         wildcard += INP_LOOKUP_MAPPED_PCB_COST;
1598 #endif
1599                                 if (inp->inp_faddr.s_addr != INADDR_ANY)
1600                                         wildcard++;
1601                                 if (inp->inp_laddr.s_addr != INADDR_ANY) {
1602                                         if (laddr.s_addr == INADDR_ANY)
1603                                                 wildcard++;
1604                                         else if (inp->inp_laddr.s_addr != laddr.s_addr)
1605                                                 continue;
1606                                 } else {
1607                                         if (laddr.s_addr != INADDR_ANY)
1608                                                 wildcard++;
1609                                 }
1610                                 if (wildcard < matchwild) {
1611                                         match = inp;
1612                                         matchwild = wildcard;
1613                                         if (matchwild == 0)
1614                                                 break;
1615                                 }
1616                         }
1617                 }
1618                 return (match);
1619         }
1620 }
1621 #undef INP_LOOKUP_MAPPED_PCB_COST
1622
1623 #ifdef PCBGROUP
1624 /*
1625  * Lookup PCB in hash list, using pcbgroup tables.
1626  */
1627 static struct inpcb *
1628 in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
1629     struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
1630     u_int lport_arg, int lookupflags, struct ifnet *ifp)
1631 {
1632         struct inpcbhead *head;
1633         struct inpcb *inp, *tmpinp;
1634         u_short fport = fport_arg, lport = lport_arg;
1635         bool locked;
1636
1637         /*
1638          * First look for an exact match.
1639          */
1640         tmpinp = NULL;
1641         INP_GROUP_LOCK(pcbgroup);
1642         head = &pcbgroup->ipg_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
1643             pcbgroup->ipg_hashmask)];
1644         LIST_FOREACH(inp, head, inp_pcbgrouphash) {
1645 #ifdef INET6
1646                 /* XXX inp locking */
1647                 if ((inp->inp_vflag & INP_IPV4) == 0)
1648                         continue;
1649 #endif
1650                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
1651                     inp->inp_laddr.s_addr == laddr.s_addr &&
1652                     inp->inp_fport == fport &&
1653                     inp->inp_lport == lport) {
1654                         /*
1655                          * XXX We should be able to directly return
1656                          * the inp here, without any checks.
1657                          * Well unless both bound with SO_REUSEPORT?
1658                          */
1659                         if (prison_flag(inp->inp_cred, PR_IP4))
1660                                 goto found;
1661                         if (tmpinp == NULL)
1662                                 tmpinp = inp;
1663                 }
1664         }
1665         if (tmpinp != NULL) {
1666                 inp = tmpinp;
1667                 goto found;
1668         }
1669
1670 #ifdef  RSS
1671         /*
1672          * For incoming connections, we may wish to do a wildcard
1673          * match for an RSS-local socket.
1674          */
1675         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1676                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1677 #ifdef INET6
1678                 struct inpcb *local_wild_mapped = NULL;
1679 #endif
1680                 struct inpcb *jail_wild = NULL;
1681                 struct inpcbhead *head;
1682                 int injail;
1683
1684                 /*
1685                  * Order of socket selection - we always prefer jails.
1686                  *      1. jailed, non-wild.
1687                  *      2. jailed, wild.
1688                  *      3. non-jailed, non-wild.
1689                  *      4. non-jailed, wild.
1690                  */
1691
1692                 head = &pcbgroup->ipg_hashbase[INP_PCBHASH(INADDR_ANY,
1693                     lport, 0, pcbgroup->ipg_hashmask)];
1694                 LIST_FOREACH(inp, head, inp_pcbgrouphash) {
1695 #ifdef INET6
1696                         /* XXX inp locking */
1697                         if ((inp->inp_vflag & INP_IPV4) == 0)
1698                                 continue;
1699 #endif
1700                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1701                             inp->inp_lport != lport)
1702                                 continue;
1703
1704                         injail = prison_flag(inp->inp_cred, PR_IP4);
1705                         if (injail) {
1706                                 if (prison_check_ip4(inp->inp_cred,
1707                                     &laddr) != 0)
1708                                         continue;
1709                         } else {
1710                                 if (local_exact != NULL)
1711                                         continue;
1712                         }
1713
1714                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1715                                 if (injail)
1716                                         goto found;
1717                                 else
1718                                         local_exact = inp;
1719                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1720 #ifdef INET6
1721                                 /* XXX inp locking, NULL check */
1722                                 if (inp->inp_vflag & INP_IPV6PROTO)
1723                                         local_wild_mapped = inp;
1724                                 else
1725 #endif
1726                                         if (injail)
1727                                                 jail_wild = inp;
1728                                         else
1729                                                 local_wild = inp;
1730                         }
1731                 } /* LIST_FOREACH */
1732
1733                 inp = jail_wild;
1734                 if (inp == NULL)
1735                         inp = local_exact;
1736                 if (inp == NULL)
1737                         inp = local_wild;
1738 #ifdef INET6
1739                 if (inp == NULL)
1740                         inp = local_wild_mapped;
1741 #endif
1742                 if (inp != NULL)
1743                         goto found;
1744         }
1745 #endif
1746
1747         /*
1748          * Then look for a wildcard match, if requested.
1749          */
1750         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1751                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1752 #ifdef INET6
1753                 struct inpcb *local_wild_mapped = NULL;
1754 #endif
1755                 struct inpcb *jail_wild = NULL;
1756                 struct inpcbhead *head;
1757                 int injail;
1758
1759                 /*
1760                  * Order of socket selection - we always prefer jails.
1761                  *      1. jailed, non-wild.
1762                  *      2. jailed, wild.
1763                  *      3. non-jailed, non-wild.
1764                  *      4. non-jailed, wild.
1765                  */
1766                 head = &pcbinfo->ipi_wildbase[INP_PCBHASH(INADDR_ANY, lport,
1767                     0, pcbinfo->ipi_wildmask)];
1768                 LIST_FOREACH(inp, head, inp_pcbgroup_wild) {
1769 #ifdef INET6
1770                         /* XXX inp locking */
1771                         if ((inp->inp_vflag & INP_IPV4) == 0)
1772                                 continue;
1773 #endif
1774                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1775                             inp->inp_lport != lport)
1776                                 continue;
1777
1778                         injail = prison_flag(inp->inp_cred, PR_IP4);
1779                         if (injail) {
1780                                 if (prison_check_ip4(inp->inp_cred,
1781                                     &laddr) != 0)
1782                                         continue;
1783                         } else {
1784                                 if (local_exact != NULL)
1785                                         continue;
1786                         }
1787
1788                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1789                                 if (injail)
1790                                         goto found;
1791                                 else
1792                                         local_exact = inp;
1793                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1794 #ifdef INET6
1795                                 /* XXX inp locking, NULL check */
1796                                 if (inp->inp_vflag & INP_IPV6PROTO)
1797                                         local_wild_mapped = inp;
1798                                 else
1799 #endif
1800                                         if (injail)
1801                                                 jail_wild = inp;
1802                                         else
1803                                                 local_wild = inp;
1804                         }
1805                 } /* LIST_FOREACH */
1806                 inp = jail_wild;
1807                 if (inp == NULL)
1808                         inp = local_exact;
1809                 if (inp == NULL)
1810                         inp = local_wild;
1811 #ifdef INET6
1812                 if (inp == NULL)
1813                         inp = local_wild_mapped;
1814 #endif
1815                 if (inp != NULL)
1816                         goto found;
1817         } /* if (lookupflags & INPLOOKUP_WILDCARD) */
1818         INP_GROUP_UNLOCK(pcbgroup);
1819         return (NULL);
1820
1821 found:
1822         if (lookupflags & INPLOOKUP_WLOCKPCB)
1823                 locked = TRY_INP_WLOCK(inp);
1824         else if (lookupflags & INPLOOKUP_RLOCKPCB)
1825                 locked = TRY_INP_RLOCK(inp);
1826         else
1827                 panic("%s: locking bug", __func__);
1828         if (!locked)
1829                 in_pcbref(inp);
1830         INP_GROUP_UNLOCK(pcbgroup);
1831         if (!locked) {
1832                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
1833                         INP_WLOCK(inp);
1834                         if (in_pcbrele_wlocked(inp))
1835                                 return (NULL);
1836                 } else {
1837                         INP_RLOCK(inp);
1838                         if (in_pcbrele_rlocked(inp))
1839                                 return (NULL);
1840                 }
1841         }
1842 #ifdef INVARIANTS
1843         if (lookupflags & INPLOOKUP_WLOCKPCB)
1844                 INP_WLOCK_ASSERT(inp);
1845         else
1846                 INP_RLOCK_ASSERT(inp);
1847 #endif
1848         return (inp);
1849 }
1850 #endif /* PCBGROUP */
1851
1852 /*
1853  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
1854  * that the caller has locked the hash list, and will not perform any further
1855  * locking or reference operations on either the hash list or the connection.
1856  */
1857 static struct inpcb *
1858 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1859     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
1860     struct ifnet *ifp)
1861 {
1862         struct inpcbhead *head;
1863         struct inpcb *inp, *tmpinp;
1864         u_short fport = fport_arg, lport = lport_arg;
1865
1866         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1867             ("%s: invalid lookup flags %d", __func__, lookupflags));
1868
1869         INP_HASH_LOCK_ASSERT(pcbinfo);
1870
1871         /*
1872          * First look for an exact match.
1873          */
1874         tmpinp = NULL;
1875         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
1876             pcbinfo->ipi_hashmask)];
1877         LIST_FOREACH(inp, head, inp_hash) {
1878 #ifdef INET6
1879                 /* XXX inp locking */
1880                 if ((inp->inp_vflag & INP_IPV4) == 0)
1881                         continue;
1882 #endif
1883                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
1884                     inp->inp_laddr.s_addr == laddr.s_addr &&
1885                     inp->inp_fport == fport &&
1886                     inp->inp_lport == lport) {
1887                         /*
1888                          * XXX We should be able to directly return
1889                          * the inp here, without any checks.
1890                          * Well unless both bound with SO_REUSEPORT?
1891                          */
1892                         if (prison_flag(inp->inp_cred, PR_IP4))
1893                                 return (inp);
1894                         if (tmpinp == NULL)
1895                                 tmpinp = inp;
1896                 }
1897         }
1898         if (tmpinp != NULL)
1899                 return (tmpinp);
1900
1901         /*
1902          * Then look for a wildcard match, if requested.
1903          */
1904         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1905                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1906 #ifdef INET6
1907                 struct inpcb *local_wild_mapped = NULL;
1908 #endif
1909                 struct inpcb *jail_wild = NULL;
1910                 int injail;
1911
1912                 /*
1913                  * Order of socket selection - we always prefer jails.
1914                  *      1. jailed, non-wild.
1915                  *      2. jailed, wild.
1916                  *      3. non-jailed, non-wild.
1917                  *      4. non-jailed, wild.
1918                  */
1919
1920                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1921                     0, pcbinfo->ipi_hashmask)];
1922                 LIST_FOREACH(inp, head, inp_hash) {
1923 #ifdef INET6
1924                         /* XXX inp locking */
1925                         if ((inp->inp_vflag & INP_IPV4) == 0)
1926                                 continue;
1927 #endif
1928                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1929                             inp->inp_lport != lport)
1930                                 continue;
1931
1932                         injail = prison_flag(inp->inp_cred, PR_IP4);
1933                         if (injail) {
1934                                 if (prison_check_ip4(inp->inp_cred,
1935                                     &laddr) != 0)
1936                                         continue;
1937                         } else {
1938                                 if (local_exact != NULL)
1939                                         continue;
1940                         }
1941
1942                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1943                                 if (injail)
1944                                         return (inp);
1945                                 else
1946                                         local_exact = inp;
1947                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1948 #ifdef INET6
1949                                 /* XXX inp locking, NULL check */
1950                                 if (inp->inp_vflag & INP_IPV6PROTO)
1951                                         local_wild_mapped = inp;
1952                                 else
1953 #endif
1954                                         if (injail)
1955                                                 jail_wild = inp;
1956                                         else
1957                                                 local_wild = inp;
1958                         }
1959                 } /* LIST_FOREACH */
1960                 if (jail_wild != NULL)
1961                         return (jail_wild);
1962                 if (local_exact != NULL)
1963                         return (local_exact);
1964                 if (local_wild != NULL)
1965                         return (local_wild);
1966 #ifdef INET6
1967                 if (local_wild_mapped != NULL)
1968                         return (local_wild_mapped);
1969 #endif
1970         } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
1971
1972         return (NULL);
1973 }
1974
1975 /*
1976  * Lookup PCB in hash list, using pcbinfo tables.  This variation locks the
1977  * hash list lock, and will return the inpcb locked (i.e., requires
1978  * INPLOOKUP_LOCKPCB).
1979  */
1980 static struct inpcb *
1981 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1982     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
1983     struct ifnet *ifp)
1984 {
1985         struct inpcb *inp;
1986         bool locked;
1987
1988         INP_HASH_RLOCK(pcbinfo);
1989         inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
1990             (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
1991         if (inp != NULL) {
1992                 if (lookupflags & INPLOOKUP_WLOCKPCB)
1993                         locked = INP_TRY_WLOCK(inp);
1994                 else if (lookupflags & INPLOOKUP_RLOCKPCB)
1995                         locked = INP_TRY_RLOCK(inp);
1996                 else
1997                         panic("%s: locking bug", __func__);
1998                 if (!locked)
1999                         in_pcbref(inp);
2000                 INP_HASH_RUNLOCK(pcbinfo);
2001                 if (!locked) {
2002                         if (lookupflags & INPLOOKUP_WLOCKPCB) {
2003                                 INP_WLOCK(inp);
2004                                 if (in_pcbrele_wlocked(inp))
2005                                         return (NULL);
2006                         } else {
2007                                 INP_RLOCK(inp);
2008                                 if (in_pcbrele_rlocked(inp))
2009                                         return (NULL);
2010                         }
2011                 }
2012 #ifdef INVARIANTS
2013                 if (lookupflags & INPLOOKUP_WLOCKPCB)
2014                         INP_WLOCK_ASSERT(inp);
2015                 else
2016                         INP_RLOCK_ASSERT(inp);
2017 #endif
2018         } else
2019                 INP_HASH_RUNLOCK(pcbinfo);
2020         return (inp);
2021 }
2022
2023 /*
2024  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2025  * from which a pre-calculated hash value may be extracted.
2026  *
2027  * Possibly more of this logic should be in in_pcbgroup.c.
2028  */
2029 struct inpcb *
2030 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2031     struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp)
2032 {
2033 #if defined(PCBGROUP) && !defined(RSS)
2034         struct inpcbgroup *pcbgroup;
2035 #endif
2036
2037         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2038             ("%s: invalid lookup flags %d", __func__, lookupflags));
2039         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2040             ("%s: LOCKPCB not set", __func__));
2041
2042         /*
2043          * When not using RSS, use connection groups in preference to the
2044          * reservation table when looking up 4-tuples.  When using RSS, just
2045          * use the reservation table, due to the cost of the Toeplitz hash
2046          * in software.
2047          *
2048          * XXXRW: This policy belongs in the pcbgroup code, as in principle
2049          * we could be doing RSS with a non-Toeplitz hash that is affordable
2050          * in software.
2051          */
2052 #if defined(PCBGROUP) && !defined(RSS)
2053         if (in_pcbgroup_enabled(pcbinfo)) {
2054                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2055                     fport);
2056                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2057                     laddr, lport, lookupflags, ifp));
2058         }
2059 #endif
2060         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2061             lookupflags, ifp));
2062 }
2063
2064 struct inpcb *
2065 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2066     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2067     struct ifnet *ifp, struct mbuf *m)
2068 {
2069 #ifdef PCBGROUP
2070         struct inpcbgroup *pcbgroup;
2071 #endif
2072
2073         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2074             ("%s: invalid lookup flags %d", __func__, lookupflags));
2075         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2076             ("%s: LOCKPCB not set", __func__));
2077
2078 #ifdef PCBGROUP
2079         /*
2080          * If we can use a hardware-generated hash to look up the connection
2081          * group, use that connection group to find the inpcb.  Otherwise
2082          * fall back on a software hash -- or the reservation table if we're
2083          * using RSS.
2084          *
2085          * XXXRW: As above, that policy belongs in the pcbgroup code.
2086          */
2087         if (in_pcbgroup_enabled(pcbinfo) &&
2088             !(M_HASHTYPE_TEST(m, M_HASHTYPE_NONE))) {
2089                 pcbgroup = in_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
2090                     m->m_pkthdr.flowid);
2091                 if (pcbgroup != NULL)
2092                         return (in_pcblookup_group(pcbinfo, pcbgroup, faddr,
2093                             fport, laddr, lport, lookupflags, ifp));
2094 #ifndef RSS
2095                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2096                     fport);
2097                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2098                     laddr, lport, lookupflags, ifp));
2099 #endif
2100         }
2101 #endif
2102         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2103             lookupflags, ifp));
2104 }
2105 #endif /* INET */
2106
2107 /*
2108  * Insert PCB onto various hash lists.
2109  */
2110 static int
2111 in_pcbinshash_internal(struct inpcb *inp, int do_pcbgroup_update)
2112 {
2113         struct inpcbhead *pcbhash;
2114         struct inpcbporthead *pcbporthash;
2115         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2116         struct inpcbport *phd;
2117         u_int32_t hashkey_faddr;
2118
2119         INP_WLOCK_ASSERT(inp);
2120         INP_HASH_WLOCK_ASSERT(pcbinfo);
2121
2122         KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2123             ("in_pcbinshash: INP_INHASHLIST"));
2124
2125 #ifdef INET6
2126         if (inp->inp_vflag & INP_IPV6)
2127                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2128         else
2129 #endif
2130         hashkey_faddr = inp->inp_faddr.s_addr;
2131
2132         pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2133                  inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2134
2135         pcbporthash = &pcbinfo->ipi_porthashbase[
2136             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2137
2138         /*
2139          * Go through port list and look for a head for this lport.
2140          */
2141         LIST_FOREACH(phd, pcbporthash, phd_hash) {
2142                 if (phd->phd_port == inp->inp_lport)
2143                         break;
2144         }
2145         /*
2146          * If none exists, malloc one and tack it on.
2147          */
2148         if (phd == NULL) {
2149                 phd = malloc(sizeof(struct inpcbport), M_PCB, M_NOWAIT);
2150                 if (phd == NULL) {
2151                         return (ENOBUFS); /* XXX */
2152                 }
2153                 phd->phd_port = inp->inp_lport;
2154                 LIST_INIT(&phd->phd_pcblist);
2155                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2156         }
2157         inp->inp_phd = phd;
2158         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2159         LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
2160         inp->inp_flags |= INP_INHASHLIST;
2161 #ifdef PCBGROUP
2162         if (do_pcbgroup_update)
2163                 in_pcbgroup_update(inp);
2164 #endif
2165         return (0);
2166 }
2167
2168 /*
2169  * For now, there are two public interfaces to insert an inpcb into the hash
2170  * lists -- one that does update pcbgroups, and one that doesn't.  The latter
2171  * is used only in the TCP syncache, where in_pcbinshash is called before the
2172  * full 4-tuple is set for the inpcb, and we don't want to install in the
2173  * pcbgroup until later.
2174  *
2175  * XXXRW: This seems like a misfeature.  in_pcbinshash should always update
2176  * connection groups, and partially initialised inpcbs should not be exposed
2177  * to either reservation hash tables or pcbgroups.
2178  */
2179 int
2180 in_pcbinshash(struct inpcb *inp)
2181 {
2182
2183         return (in_pcbinshash_internal(inp, 1));
2184 }
2185
2186 int
2187 in_pcbinshash_nopcbgroup(struct inpcb *inp)
2188 {
2189
2190         return (in_pcbinshash_internal(inp, 0));
2191 }
2192
2193 /*
2194  * Move PCB to the proper hash bucket when { faddr, fport } have  been
2195  * changed. NOTE: This does not handle the case of the lport changing (the
2196  * hashed port list would have to be updated as well), so the lport must
2197  * not change after in_pcbinshash() has been called.
2198  */
2199 void
2200 in_pcbrehash_mbuf(struct inpcb *inp, struct mbuf *m)
2201 {
2202         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2203         struct inpcbhead *head;
2204         u_int32_t hashkey_faddr;
2205
2206         INP_WLOCK_ASSERT(inp);
2207         INP_HASH_WLOCK_ASSERT(pcbinfo);
2208
2209         KASSERT(inp->inp_flags & INP_INHASHLIST,
2210             ("in_pcbrehash: !INP_INHASHLIST"));
2211
2212 #ifdef INET6
2213         if (inp->inp_vflag & INP_IPV6)
2214                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2215         else
2216 #endif
2217         hashkey_faddr = inp->inp_faddr.s_addr;
2218
2219         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2220                 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2221
2222         LIST_REMOVE(inp, inp_hash);
2223         LIST_INSERT_HEAD(head, inp, inp_hash);
2224
2225 #ifdef PCBGROUP
2226         if (m != NULL)
2227                 in_pcbgroup_update_mbuf(inp, m);
2228         else
2229                 in_pcbgroup_update(inp);
2230 #endif
2231 }
2232
2233 void
2234 in_pcbrehash(struct inpcb *inp)
2235 {
2236
2237         in_pcbrehash_mbuf(inp, NULL);
2238 }
2239
2240 /*
2241  * Remove PCB from various lists.
2242  */
2243 static void
2244 in_pcbremlists(struct inpcb *inp)
2245 {
2246         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2247
2248 #ifdef INVARIANTS
2249         if (pcbinfo == &V_tcbinfo) {
2250                 INP_INFO_RLOCK_ASSERT(pcbinfo);
2251         } else {
2252                 INP_INFO_WLOCK_ASSERT(pcbinfo);
2253         }
2254 #endif
2255
2256         INP_WLOCK_ASSERT(inp);
2257         INP_LIST_WLOCK_ASSERT(pcbinfo);
2258
2259         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
2260         if (inp->inp_flags & INP_INHASHLIST) {
2261                 struct inpcbport *phd = inp->inp_phd;
2262
2263                 INP_HASH_WLOCK(pcbinfo);
2264                 LIST_REMOVE(inp, inp_hash);
2265                 LIST_REMOVE(inp, inp_portlist);
2266                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
2267                         LIST_REMOVE(phd, phd_hash);
2268                         free(phd, M_PCB);
2269                 }
2270                 INP_HASH_WUNLOCK(pcbinfo);
2271                 inp->inp_flags &= ~INP_INHASHLIST;
2272         }
2273         LIST_REMOVE(inp, inp_list);
2274         pcbinfo->ipi_count--;
2275 #ifdef PCBGROUP
2276         in_pcbgroup_remove(inp);
2277 #endif
2278 }
2279
2280 /*
2281  * Check for alternatives when higher level complains
2282  * about service problems.  For now, invalidate cached
2283  * routing information.  If the route was created dynamically
2284  * (by a redirect), time to try a default gateway again.
2285  */
2286 void
2287 in_losing(struct inpcb *inp)
2288 {
2289
2290         RO_INVALIDATE_CACHE(&inp->inp_route);
2291         return;
2292 }
2293
2294 /*
2295  * A set label operation has occurred at the socket layer, propagate the
2296  * label change into the in_pcb for the socket.
2297  */
2298 void
2299 in_pcbsosetlabel(struct socket *so)
2300 {
2301 #ifdef MAC
2302         struct inpcb *inp;
2303
2304         inp = sotoinpcb(so);
2305         KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2306
2307         INP_WLOCK(inp);
2308         SOCK_LOCK(so);
2309         mac_inpcb_sosetlabel(so, inp);
2310         SOCK_UNLOCK(so);
2311         INP_WUNLOCK(inp);
2312 #endif
2313 }
2314
2315 /*
2316  * ipport_tick runs once per second, determining if random port allocation
2317  * should be continued.  If more than ipport_randomcps ports have been
2318  * allocated in the last second, then we return to sequential port
2319  * allocation. We return to random allocation only once we drop below
2320  * ipport_randomcps for at least ipport_randomtime seconds.
2321  */
2322 static void
2323 ipport_tick(void *xtp)
2324 {
2325         VNET_ITERATOR_DECL(vnet_iter);
2326
2327         VNET_LIST_RLOCK_NOSLEEP();
2328         VNET_FOREACH(vnet_iter) {
2329                 CURVNET_SET(vnet_iter); /* XXX appease INVARIANTS here */
2330                 if (V_ipport_tcpallocs <=
2331                     V_ipport_tcplastcount + V_ipport_randomcps) {
2332                         if (V_ipport_stoprandom > 0)
2333                                 V_ipport_stoprandom--;
2334                 } else
2335                         V_ipport_stoprandom = V_ipport_randomtime;
2336                 V_ipport_tcplastcount = V_ipport_tcpallocs;
2337                 CURVNET_RESTORE();
2338         }
2339         VNET_LIST_RUNLOCK_NOSLEEP();
2340         callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
2341 }
2342
2343 static void
2344 ip_fini(void *xtp)
2345 {
2346
2347         callout_stop(&ipport_tick_callout);
2348 }
2349
2350 /*
2351  * The ipport_callout should start running at about the time we attach the
2352  * inet or inet6 domains.
2353  */
2354 static void
2355 ipport_tick_init(const void *unused __unused)
2356 {
2357
2358         /* Start ipport_tick. */
2359         callout_init(&ipport_tick_callout, 1);
2360         callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
2361         EVENTHANDLER_REGISTER(shutdown_pre_sync, ip_fini, NULL,
2362                 SHUTDOWN_PRI_DEFAULT);
2363 }
2364 SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
2365     ipport_tick_init, NULL);
2366
2367 void
2368 inp_wlock(struct inpcb *inp)
2369 {
2370
2371         INP_WLOCK(inp);
2372 }
2373
2374 void
2375 inp_wunlock(struct inpcb *inp)
2376 {
2377
2378         INP_WUNLOCK(inp);
2379 }
2380
2381 void
2382 inp_rlock(struct inpcb *inp)
2383 {
2384
2385         INP_RLOCK(inp);
2386 }
2387
2388 void
2389 inp_runlock(struct inpcb *inp)
2390 {
2391
2392         INP_RUNLOCK(inp);
2393 }
2394
2395 #ifdef INVARIANT_SUPPORT
2396 void
2397 inp_lock_assert(struct inpcb *inp)
2398 {
2399
2400         INP_WLOCK_ASSERT(inp);
2401 }
2402
2403 void
2404 inp_unlock_assert(struct inpcb *inp)
2405 {
2406
2407         INP_UNLOCK_ASSERT(inp);
2408 }
2409 #endif
2410
2411 void
2412 inp_apply_all(void (*func)(struct inpcb *, void *), void *arg)
2413 {
2414         struct inpcb *inp;
2415
2416         INP_INFO_WLOCK(&V_tcbinfo);
2417         LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) {
2418                 INP_WLOCK(inp);
2419                 func(inp, arg);
2420                 INP_WUNLOCK(inp);
2421         }
2422         INP_INFO_WUNLOCK(&V_tcbinfo);
2423 }
2424
2425 struct socket *
2426 inp_inpcbtosocket(struct inpcb *inp)
2427 {
2428
2429         INP_WLOCK_ASSERT(inp);
2430         return (inp->inp_socket);
2431 }
2432
2433 struct tcpcb *
2434 inp_inpcbtotcpcb(struct inpcb *inp)
2435 {
2436
2437         INP_WLOCK_ASSERT(inp);
2438         return ((struct tcpcb *)inp->inp_ppcb);
2439 }
2440
2441 int
2442 inp_ip_tos_get(const struct inpcb *inp)
2443 {
2444
2445         return (inp->inp_ip_tos);
2446 }
2447
2448 void
2449 inp_ip_tos_set(struct inpcb *inp, int val)
2450 {
2451
2452         inp->inp_ip_tos = val;
2453 }
2454
2455 void
2456 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2457     uint32_t *faddr, uint16_t *fp)
2458 {
2459
2460         INP_LOCK_ASSERT(inp);
2461         *laddr = inp->inp_laddr.s_addr;
2462         *faddr = inp->inp_faddr.s_addr;
2463         *lp = inp->inp_lport;
2464         *fp = inp->inp_fport;
2465 }
2466
2467 struct inpcb *
2468 so_sotoinpcb(struct socket *so)
2469 {
2470
2471         return (sotoinpcb(so));
2472 }
2473
2474 struct tcpcb *
2475 so_sototcpcb(struct socket *so)
2476 {
2477
2478         return (sototcpcb(so));
2479 }
2480
2481 /*
2482  * Create an external-format (``xinpcb'') structure using the information in
2483  * the kernel-format in_pcb structure pointed to by inp.  This is done to
2484  * reduce the spew of irrelevant information over this interface, to isolate
2485  * user code from changes in the kernel structure, and potentially to provide
2486  * information-hiding if we decide that some of this information should be
2487  * hidden from users.
2488  */
2489 void
2490 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2491 {
2492
2493         xi->xi_len = sizeof(struct xinpcb);
2494         if (inp->inp_socket)
2495                 sotoxsocket(inp->inp_socket, &xi->xi_socket);
2496         else
2497                 bzero(&xi->xi_socket, sizeof(struct xsocket));
2498         bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2499         xi->inp_gencnt = inp->inp_gencnt;
2500         xi->inp_ppcb = inp->inp_ppcb;
2501         xi->inp_flow = inp->inp_flow;
2502         xi->inp_flowid = inp->inp_flowid;
2503         xi->inp_flowtype = inp->inp_flowtype;
2504         xi->inp_flags = inp->inp_flags;
2505         xi->inp_flags2 = inp->inp_flags2;
2506         xi->inp_rss_listen_bucket = inp->inp_rss_listen_bucket;
2507         xi->in6p_cksum = inp->in6p_cksum;
2508         xi->in6p_hops = inp->in6p_hops;
2509         xi->inp_ip_tos = inp->inp_ip_tos;
2510         xi->inp_vflag = inp->inp_vflag;
2511         xi->inp_ip_ttl = inp->inp_ip_ttl;
2512         xi->inp_ip_p = inp->inp_ip_p;
2513         xi->inp_ip_minttl = inp->inp_ip_minttl;
2514 }
2515
2516 #ifdef DDB
2517 static void
2518 db_print_indent(int indent)
2519 {
2520         int i;
2521
2522         for (i = 0; i < indent; i++)
2523                 db_printf(" ");
2524 }
2525
2526 static void
2527 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
2528 {
2529         char faddr_str[48], laddr_str[48];
2530
2531         db_print_indent(indent);
2532         db_printf("%s at %p\n", name, inc);
2533
2534         indent += 2;
2535
2536 #ifdef INET6
2537         if (inc->inc_flags & INC_ISIPV6) {
2538                 /* IPv6. */
2539                 ip6_sprintf(laddr_str, &inc->inc6_laddr);
2540                 ip6_sprintf(faddr_str, &inc->inc6_faddr);
2541         } else
2542 #endif
2543         {
2544                 /* IPv4. */
2545                 inet_ntoa_r(inc->inc_laddr, laddr_str);
2546                 inet_ntoa_r(inc->inc_faddr, faddr_str);
2547         }
2548         db_print_indent(indent);
2549         db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
2550             ntohs(inc->inc_lport));
2551         db_print_indent(indent);
2552         db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
2553             ntohs(inc->inc_fport));
2554 }
2555
2556 static void
2557 db_print_inpflags(int inp_flags)
2558 {
2559         int comma;
2560
2561         comma = 0;
2562         if (inp_flags & INP_RECVOPTS) {
2563                 db_printf("%sINP_RECVOPTS", comma ? ", " : "");
2564                 comma = 1;
2565         }
2566         if (inp_flags & INP_RECVRETOPTS) {
2567                 db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
2568                 comma = 1;
2569         }
2570         if (inp_flags & INP_RECVDSTADDR) {
2571                 db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
2572                 comma = 1;
2573         }
2574         if (inp_flags & INP_ORIGDSTADDR) {
2575                 db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
2576                 comma = 1;
2577         }
2578         if (inp_flags & INP_HDRINCL) {
2579                 db_printf("%sINP_HDRINCL", comma ? ", " : "");
2580                 comma = 1;
2581         }
2582         if (inp_flags & INP_HIGHPORT) {
2583                 db_printf("%sINP_HIGHPORT", comma ? ", " : "");
2584                 comma = 1;
2585         }
2586         if (inp_flags & INP_LOWPORT) {
2587                 db_printf("%sINP_LOWPORT", comma ? ", " : "");
2588                 comma = 1;
2589         }
2590         if (inp_flags & INP_ANONPORT) {
2591                 db_printf("%sINP_ANONPORT", comma ? ", " : "");
2592                 comma = 1;
2593         }
2594         if (inp_flags & INP_RECVIF) {
2595                 db_printf("%sINP_RECVIF", comma ? ", " : "");
2596                 comma = 1;
2597         }
2598         if (inp_flags & INP_MTUDISC) {
2599                 db_printf("%sINP_MTUDISC", comma ? ", " : "");
2600                 comma = 1;
2601         }
2602         if (inp_flags & INP_RECVTTL) {
2603                 db_printf("%sINP_RECVTTL", comma ? ", " : "");
2604                 comma = 1;
2605         }
2606         if (inp_flags & INP_DONTFRAG) {
2607                 db_printf("%sINP_DONTFRAG", comma ? ", " : "");
2608                 comma = 1;
2609         }
2610         if (inp_flags & INP_RECVTOS) {
2611                 db_printf("%sINP_RECVTOS", comma ? ", " : "");
2612                 comma = 1;
2613         }
2614         if (inp_flags & IN6P_IPV6_V6ONLY) {
2615                 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
2616                 comma = 1;
2617         }
2618         if (inp_flags & IN6P_PKTINFO) {
2619                 db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
2620                 comma = 1;
2621         }
2622         if (inp_flags & IN6P_HOPLIMIT) {
2623                 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
2624                 comma = 1;
2625         }
2626         if (inp_flags & IN6P_HOPOPTS) {
2627                 db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
2628                 comma = 1;
2629         }
2630         if (inp_flags & IN6P_DSTOPTS) {
2631                 db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
2632                 comma = 1;
2633         }
2634         if (inp_flags & IN6P_RTHDR) {
2635                 db_printf("%sIN6P_RTHDR", comma ? ", " : "");
2636                 comma = 1;
2637         }
2638         if (inp_flags & IN6P_RTHDRDSTOPTS) {
2639                 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
2640                 comma = 1;
2641         }
2642         if (inp_flags & IN6P_TCLASS) {
2643                 db_printf("%sIN6P_TCLASS", comma ? ", " : "");
2644                 comma = 1;
2645         }
2646         if (inp_flags & IN6P_AUTOFLOWLABEL) {
2647                 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
2648                 comma = 1;
2649         }
2650         if (inp_flags & INP_TIMEWAIT) {
2651                 db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
2652                 comma  = 1;
2653         }
2654         if (inp_flags & INP_ONESBCAST) {
2655                 db_printf("%sINP_ONESBCAST", comma ? ", " : "");
2656                 comma  = 1;
2657         }
2658         if (inp_flags & INP_DROPPED) {
2659                 db_printf("%sINP_DROPPED", comma ? ", " : "");
2660                 comma  = 1;
2661         }
2662         if (inp_flags & INP_SOCKREF) {
2663                 db_printf("%sINP_SOCKREF", comma ? ", " : "");
2664                 comma  = 1;
2665         }
2666         if (inp_flags & IN6P_RFC2292) {
2667                 db_printf("%sIN6P_RFC2292", comma ? ", " : "");
2668                 comma = 1;
2669         }
2670         if (inp_flags & IN6P_MTU) {
2671                 db_printf("IN6P_MTU%s", comma ? ", " : "");
2672                 comma = 1;
2673         }
2674 }
2675
2676 static void
2677 db_print_inpvflag(u_char inp_vflag)
2678 {
2679         int comma;
2680
2681         comma = 0;
2682         if (inp_vflag & INP_IPV4) {
2683                 db_printf("%sINP_IPV4", comma ? ", " : "");
2684                 comma  = 1;
2685         }
2686         if (inp_vflag & INP_IPV6) {
2687                 db_printf("%sINP_IPV6", comma ? ", " : "");
2688                 comma  = 1;
2689         }
2690         if (inp_vflag & INP_IPV6PROTO) {
2691                 db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
2692                 comma  = 1;
2693         }
2694 }
2695
2696 static void
2697 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
2698 {
2699
2700         db_print_indent(indent);
2701         db_printf("%s at %p\n", name, inp);
2702
2703         indent += 2;
2704
2705         db_print_indent(indent);
2706         db_printf("inp_flow: 0x%x\n", inp->inp_flow);
2707
2708         db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
2709
2710         db_print_indent(indent);
2711         db_printf("inp_ppcb: %p   inp_pcbinfo: %p   inp_socket: %p\n",
2712             inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
2713
2714         db_print_indent(indent);
2715         db_printf("inp_label: %p   inp_flags: 0x%x (",
2716            inp->inp_label, inp->inp_flags);
2717         db_print_inpflags(inp->inp_flags);
2718         db_printf(")\n");
2719
2720         db_print_indent(indent);
2721         db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
2722             inp->inp_vflag);
2723         db_print_inpvflag(inp->inp_vflag);
2724         db_printf(")\n");
2725
2726         db_print_indent(indent);
2727         db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
2728             inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
2729
2730         db_print_indent(indent);
2731 #ifdef INET6
2732         if (inp->inp_vflag & INP_IPV6) {
2733                 db_printf("in6p_options: %p   in6p_outputopts: %p   "
2734                     "in6p_moptions: %p\n", inp->in6p_options,
2735                     inp->in6p_outputopts, inp->in6p_moptions);
2736                 db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
2737                     "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
2738                     inp->in6p_hops);
2739         } else
2740 #endif
2741         {
2742                 db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
2743                     "inp_ip_moptions: %p\n", inp->inp_ip_tos,
2744                     inp->inp_options, inp->inp_moptions);
2745         }
2746
2747         db_print_indent(indent);
2748         db_printf("inp_phd: %p   inp_gencnt: %ju\n", inp->inp_phd,
2749             (uintmax_t)inp->inp_gencnt);
2750 }
2751
2752 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
2753 {
2754         struct inpcb *inp;
2755
2756         if (!have_addr) {
2757                 db_printf("usage: show inpcb <addr>\n");
2758                 return;
2759         }
2760         inp = (struct inpcb *)addr;
2761
2762         db_print_inpcb(inp, "inpcb", 0);
2763 }
2764 #endif /* DDB */
2765
2766 #ifdef RATELIMIT
2767 /*
2768  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
2769  * if any.
2770  */
2771 int
2772 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
2773 {
2774         union if_snd_tag_modify_params params = {
2775                 .rate_limit.max_rate = max_pacing_rate,
2776         };
2777         struct m_snd_tag *mst;
2778         struct ifnet *ifp;
2779         int error;
2780
2781         mst = inp->inp_snd_tag;
2782         if (mst == NULL)
2783                 return (EINVAL);
2784
2785         ifp = mst->ifp;
2786         if (ifp == NULL)
2787                 return (EINVAL);
2788
2789         if (ifp->if_snd_tag_modify == NULL) {
2790                 error = EOPNOTSUPP;
2791         } else {
2792                 error = ifp->if_snd_tag_modify(mst, &params);
2793         }
2794         return (error);
2795 }
2796
2797 /*
2798  * Query existing TX rate limit based on the existing
2799  * "inp->inp_snd_tag", if any.
2800  */
2801 int
2802 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
2803 {
2804         union if_snd_tag_query_params params = { };
2805         struct m_snd_tag *mst;
2806         struct ifnet *ifp;
2807         int error;
2808
2809         mst = inp->inp_snd_tag;
2810         if (mst == NULL)
2811                 return (EINVAL);
2812
2813         ifp = mst->ifp;
2814         if (ifp == NULL)
2815                 return (EINVAL);
2816
2817         if (ifp->if_snd_tag_query == NULL) {
2818                 error = EOPNOTSUPP;
2819         } else {
2820                 error = ifp->if_snd_tag_query(mst, &params);
2821                 if (error == 0 &&  p_max_pacing_rate != NULL)
2822                         *p_max_pacing_rate = params.rate_limit.max_rate;
2823         }
2824         return (error);
2825 }
2826
2827 /*
2828  * Query existing TX queue level based on the existing
2829  * "inp->inp_snd_tag", if any.
2830  */
2831 int
2832 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
2833 {
2834         union if_snd_tag_query_params params = { };
2835         struct m_snd_tag *mst;
2836         struct ifnet *ifp;
2837         int error;
2838
2839         mst = inp->inp_snd_tag;
2840         if (mst == NULL)
2841                 return (EINVAL);
2842
2843         ifp = mst->ifp;
2844         if (ifp == NULL)
2845                 return (EINVAL);
2846
2847         if (ifp->if_snd_tag_query == NULL)
2848                 return (EOPNOTSUPP);
2849
2850         error = ifp->if_snd_tag_query(mst, &params);
2851         if (error == 0 &&  p_txqueue_level != NULL)
2852                 *p_txqueue_level = params.rate_limit.queue_level;
2853         return (error);
2854 }
2855
2856 /*
2857  * Allocate a new TX rate limit send tag from the network interface
2858  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
2859  */
2860 int
2861 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
2862     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate)
2863 {
2864         union if_snd_tag_alloc_params params = {
2865                 .rate_limit.hdr.type = (max_pacing_rate == -1U) ?
2866                     IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
2867                 .rate_limit.hdr.flowid = flowid,
2868                 .rate_limit.hdr.flowtype = flowtype,
2869                 .rate_limit.max_rate = max_pacing_rate,
2870         };
2871         int error;
2872
2873         INP_WLOCK_ASSERT(inp);
2874
2875         if (inp->inp_snd_tag != NULL)
2876                 return (EINVAL);
2877
2878         if (ifp->if_snd_tag_alloc == NULL) {
2879                 error = EOPNOTSUPP;
2880         } else {
2881                 error = ifp->if_snd_tag_alloc(ifp, &params, &inp->inp_snd_tag);
2882
2883                 /*
2884                  * At success increment the refcount on
2885                  * the send tag's network interface:
2886                  */
2887                 if (error == 0)
2888                         if_ref(inp->inp_snd_tag->ifp);
2889         }
2890         return (error);
2891 }
2892
2893 /*
2894  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
2895  * if any:
2896  */
2897 void
2898 in_pcbdetach_txrtlmt(struct inpcb *inp)
2899 {
2900         struct m_snd_tag *mst;
2901         struct ifnet *ifp;
2902
2903         INP_WLOCK_ASSERT(inp);
2904
2905         mst = inp->inp_snd_tag;
2906         inp->inp_snd_tag = NULL;
2907
2908         if (mst == NULL)
2909                 return;
2910
2911         ifp = mst->ifp;
2912         if (ifp == NULL)
2913                 return;
2914
2915         /*
2916          * If the device was detached while we still had reference(s)
2917          * on the ifp, we assume if_snd_tag_free() was replaced with
2918          * stubs.
2919          */
2920         ifp->if_snd_tag_free(mst);
2921
2922         /* release reference count on network interface */
2923         if_rele(ifp);
2924 }
2925
2926 /*
2927  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
2928  * is set in the fast path and will attach/detach/modify the TX rate
2929  * limit send tag based on the socket's so_max_pacing_rate value.
2930  */
2931 void
2932 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
2933 {
2934         struct socket *socket;
2935         uint32_t max_pacing_rate;
2936         bool did_upgrade;
2937         int error;
2938
2939         if (inp == NULL)
2940                 return;
2941
2942         socket = inp->inp_socket;
2943         if (socket == NULL)
2944                 return;
2945
2946         if (!INP_WLOCKED(inp)) {
2947                 /*
2948                  * NOTE: If the write locking fails, we need to bail
2949                  * out and use the non-ratelimited ring for the
2950                  * transmit until there is a new chance to get the
2951                  * write lock.
2952                  */
2953                 if (!INP_TRY_UPGRADE(inp))
2954                         return;
2955                 did_upgrade = 1;
2956         } else {
2957                 did_upgrade = 0;
2958         }
2959
2960         /*
2961          * NOTE: The so_max_pacing_rate value is read unlocked,
2962          * because atomic updates are not required since the variable
2963          * is checked at every mbuf we send. It is assumed that the
2964          * variable read itself will be atomic.
2965          */
2966         max_pacing_rate = socket->so_max_pacing_rate;
2967
2968         /*
2969          * NOTE: When attaching to a network interface a reference is
2970          * made to ensure the network interface doesn't go away until
2971          * all ratelimit connections are gone. The network interface
2972          * pointers compared below represent valid network interfaces,
2973          * except when comparing towards NULL.
2974          */
2975         if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
2976                 error = 0;
2977         } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
2978                 if (inp->inp_snd_tag != NULL)
2979                         in_pcbdetach_txrtlmt(inp);
2980                 error = 0;
2981         } else if (inp->inp_snd_tag == NULL) {
2982                 /*
2983                  * In order to utilize packet pacing with RSS, we need
2984                  * to wait until there is a valid RSS hash before we
2985                  * can proceed:
2986                  */
2987                 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
2988                         error = EAGAIN;
2989                 } else {
2990                         error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
2991                             mb->m_pkthdr.flowid, max_pacing_rate);
2992                 }
2993         } else {
2994                 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
2995         }
2996         if (error == 0 || error == EOPNOTSUPP)
2997                 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
2998         if (did_upgrade)
2999                 INP_DOWNGRADE(inp);
3000 }
3001
3002 /*
3003  * Track route changes for TX rate limiting.
3004  */
3005 void
3006 in_pcboutput_eagain(struct inpcb *inp)
3007 {
3008         struct socket *socket;
3009         bool did_upgrade;
3010
3011         if (inp == NULL)
3012                 return;
3013
3014         socket = inp->inp_socket;
3015         if (socket == NULL)
3016                 return;
3017
3018         if (inp->inp_snd_tag == NULL)
3019                 return;
3020
3021         if (!INP_WLOCKED(inp)) {
3022                 /*
3023                  * NOTE: If the write locking fails, we need to bail
3024                  * out and use the non-ratelimited ring for the
3025                  * transmit until there is a new chance to get the
3026                  * write lock.
3027                  */
3028                 if (!INP_TRY_UPGRADE(inp))
3029                         return;
3030                 did_upgrade = 1;
3031         } else {
3032                 did_upgrade = 0;
3033         }
3034
3035         /* detach rate limiting */
3036         in_pcbdetach_txrtlmt(inp);
3037
3038         /* make sure new mbuf send tag allocation is made */
3039         inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3040
3041         if (did_upgrade)
3042                 INP_DOWNGRADE(inp);
3043 }
3044 #endif /* RATELIMIT */