sys/netinet/in_pcb.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 1982, 1986, 1991, 1993, 1995
   5  *      The Regents of the University of California.
   6  * Copyright (c) 2007-2009 Robert N. M. Watson
   7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
   8  * All rights reserved.
   9  *
  10  * Portions of this software were developed by Robert N. M. Watson under
  11  * contract to Juniper Networks, Inc.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  * 3. Neither the name of the University nor the names of its contributors
  22  *    may be used to endorse or promote products derived from this software
  23  *    without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  *
  37  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  38  */
  39
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include "opt_ddb.h"
  44 #include "opt_ipsec.h"
  45 #include "opt_inet.h"
  46 #include "opt_inet6.h"
  47 #include "opt_ratelimit.h"
  48 #include "opt_pcbgroup.h"
  49 #include "opt_rss.h"
  50
  51 #include <sys/param.h>
  52 #include <sys/systm.h>
  53 #include <sys/lock.h>
  54 #include <sys/malloc.h>
  55 #include <sys/mbuf.h>
  56 #include <sys/callout.h>
  57 #include <sys/eventhandler.h>
  58 #include <sys/domain.h>
  59 #include <sys/protosw.h>
  60 #include <sys/rmlock.h>
  61 #include <sys/socket.h>
  62 #include <sys/socketvar.h>
  63 #include <sys/sockio.h>
  64 #include <sys/priv.h>
  65 #include <sys/proc.h>
  66 #include <sys/refcount.h>
  67 #include <sys/jail.h>
  68 #include <sys/kernel.h>
  69 #include <sys/sysctl.h>
  70
  71 #ifdef DDB
  72 #include <ddb/ddb.h>
  73 #endif
  74
  75 #include <vm/uma.h>
  76
  77 #include <net/if.h>
  78 #include <net/if_var.h>
  79 #include <net/if_types.h>
  80 #include <net/if_llatbl.h>
  81 #include <net/route.h>
  82 #include <net/rss_config.h>
  83 #include <net/vnet.h>
  84
  85 #if defined(INET) || defined(INET6)
  86 #include <netinet/in.h>
  87 #include <netinet/in_pcb.h>
  88 #include <netinet/ip_var.h>
  89 #include <netinet/tcp_var.h>
  90 #include <netinet/udp.h>
  91 #include <netinet/udp_var.h>
  92 #endif
  93 #ifdef INET
  94 #include <netinet/in_var.h>
  95 #endif
  96 #ifdef INET6
  97 #include <netinet/ip6.h>
  98 #include <netinet6/in6_pcb.h>
  99 #include <netinet6/in6_var.h>
 100 #include <netinet6/ip6_var.h>
 101 #endif /* INET6 */
 102
 103 #include <netipsec/ipsec_support.h>
 104
 105 #include <security/mac/mac_framework.h>
 106
 107 static struct callout   ipport_tick_callout;
 108
 109 /*
 110  * These configure the range of local port addresses assigned to
 111  * "unspecified" outgoing connections/packets/whatever.
 112  */
 113 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;    /* 1023 */
 114 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;    /* 600 */
 115 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;     /* 10000 */
 116 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;       /* 65535 */
 117 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;      /* 49152 */
 118 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;        /* 65535 */
 119
 120 /*
 121  * Reserved ports accessible only to root. There are significant
 122  * security considerations that must be accounted for when changing these,
 123  * but the security benefits can be great. Please be careful.
 124  */
 125 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;    /* 1023 */
 126 VNET_DEFINE(int, ipport_reservedlow);
 127
 128 /* Variables dealing with random ephemeral port allocation. */
 129 VNET_DEFINE(int, ipport_randomized) = 1;        /* user controlled via sysctl */
 130 VNET_DEFINE(int, ipport_randomcps) = 10;        /* user controlled via sysctl */
 131 VNET_DEFINE(int, ipport_randomtime) = 45;       /* user controlled via sysctl */
 132 VNET_DEFINE(int, ipport_stoprandom);            /* toggled by ipport_tick */
 133 VNET_DEFINE(int, ipport_tcpallocs);
 134 static VNET_DEFINE(int, ipport_tcplastcount);
 135
 136 #define V_ipport_tcplastcount           VNET(ipport_tcplastcount)
 137
 138 static void     in_pcbremlists(struct inpcb *inp);
 139 #ifdef INET
 140 static struct inpcb     *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
 141                             struct in_addr faddr, u_int fport_arg,
 142                             struct in_addr laddr, u_int lport_arg,
 143                             int lookupflags, struct ifnet *ifp);
 144
 145 #define RANGECHK(var, min, max) \
 146         if ((var) < (min)) { (var) = (min); } \
 147         else if ((var) > (max)) { (var) = (max); }
 148
 149 static int
 150 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 151 {
 152         int error;
 153
 154         error = sysctl_handle_int(oidp, arg1, arg2, req);
 155         if (error == 0) {
 156                 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 157                 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 158                 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
 159                 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
 160                 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
 161                 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
 162         }
 163         return (error);
 164 }
 165
 166 #undef RANGECHK
 167
 168 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0,
 169     "IP Ports");
 170
 171 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
 172         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 173         &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I", "");
 174 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
 175         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 176         &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I", "");
 177 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
 178         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 179         &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I", "");
 180 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
 181         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 182         &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I", "");
 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
 184         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 185         &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I", "");
 186 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
 187         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 188         &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I", "");
 189 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
 190         CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
 191         &VNET_NAME(ipport_reservedhigh), 0, "");
 192 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
 193         CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
 194 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
 195         CTLFLAG_VNET | CTLFLAG_RW,
 196         &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
 197 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
 198         CTLFLAG_VNET | CTLFLAG_RW,
 199         &VNET_NAME(ipport_randomcps), 0, "Maximum number of random port "
 200         "allocations before switching to a sequental one");
 201 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
 202         CTLFLAG_VNET | CTLFLAG_RW,
 203         &VNET_NAME(ipport_randomtime), 0,
 204         "Minimum time to keep sequental port "
 205         "allocation before switching to a random one");
 206 #endif /* INET */
 207
 208 /*
 209  * in_pcb.c: manage the Protocol Control Blocks.
 210  *
 211  * NOTE: It is assumed that most of these functions will be called with
 212  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
 213  * functions often modify hash chains or addresses in pcbs.
 214  */
 215
 216 /*
 217  * Different protocols initialize their inpcbs differently - giving
 218  * different name to the lock.  But they all are disposed the same.
 219  */
 220 static void
 221 inpcb_fini(void *mem, int size)
 222 {
 223         struct inpcb *inp = mem;
 224
 225         INP_LOCK_DESTROY(inp);
 226 }
 227
 228 /*
 229  * Initialize an inpcbinfo -- we should be able to reduce the number of
 230  * arguments in time.
 231  */
 232 void
 233 in_pcbinfo_init(struct inpcbinfo *pcbinfo, const char *name,
 234     struct inpcbhead *listhead, int hash_nelements, int porthash_nelements,
 235     char *inpcbzone_name, uma_init inpcbzone_init, u_int hashfields)
 236 {
 237
 238         INP_INFO_LOCK_INIT(pcbinfo, name);
 239         INP_HASH_LOCK_INIT(pcbinfo, "pcbinfohash");     /* XXXRW: argument? */
 240         INP_LIST_LOCK_INIT(pcbinfo, "pcbinfolist");
 241 #ifdef VIMAGE
 242         pcbinfo->ipi_vnet = curvnet;
 243 #endif
 244         pcbinfo->ipi_listhead = listhead;
 245         LIST_INIT(pcbinfo->ipi_listhead);
 246         pcbinfo->ipi_count = 0;
 247         pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB,
 248             &pcbinfo->ipi_hashmask);
 249         pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
 250             &pcbinfo->ipi_porthashmask);
 251 #ifdef PCBGROUP
 252         in_pcbgroup_init(pcbinfo, hashfields, hash_nelements);
 253 #endif
 254         pcbinfo->ipi_zone = uma_zcreate(inpcbzone_name, sizeof(struct inpcb),
 255             NULL, NULL, inpcbzone_init, inpcb_fini, UMA_ALIGN_PTR, 0);
 256         uma_zone_set_max(pcbinfo->ipi_zone, maxsockets);
 257         uma_zone_set_warning(pcbinfo->ipi_zone,
 258             "kern.ipc.maxsockets limit reached");
 259 }
 260
 261 /*
 262  * Destroy an inpcbinfo.
 263  */
 264 void
 265 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
 266 {
 267
 268         KASSERT(pcbinfo->ipi_count == 0,
 269             ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
 270
 271         hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask);
 272         hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
 273             pcbinfo->ipi_porthashmask);
 274 #ifdef PCBGROUP
 275         in_pcbgroup_destroy(pcbinfo);
 276 #endif
 277         uma_zdestroy(pcbinfo->ipi_zone);
 278         INP_LIST_LOCK_DESTROY(pcbinfo);
 279         INP_HASH_LOCK_DESTROY(pcbinfo);
 280         INP_INFO_LOCK_DESTROY(pcbinfo);
 281 }
 282
 283 /*
 284  * Allocate a PCB and associate it with the socket.
 285  * On success return with the PCB locked.
 286  */
 287 int
 288 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 289 {
 290         struct inpcb *inp;
 291         int error;
 292
 293 #ifdef INVARIANTS
 294         if (pcbinfo == &V_tcbinfo) {
 295                 INP_INFO_RLOCK_ASSERT(pcbinfo);
 296         } else {
 297                 INP_INFO_WLOCK_ASSERT(pcbinfo);
 298         }
 299 #endif
 300
 301         error = 0;
 302         inp = uma_zalloc(pcbinfo->ipi_zone, M_NOWAIT);
 303         if (inp == NULL)
 304                 return (ENOBUFS);
 305         bzero(&inp->inp_start_zero, inp_zero_size);
 306         inp->inp_pcbinfo = pcbinfo;
 307         inp->inp_socket = so;
 308         inp->inp_cred = crhold(so->so_cred);
 309         inp->inp_inc.inc_fibnum = so->so_fibnum;
 310 #ifdef MAC
 311         error = mac_inpcb_init(inp, M_NOWAIT);
 312         if (error != 0)
 313                 goto out;
 314         mac_inpcb_create(so, inp);
 315 #endif
 316 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
 317         error = ipsec_init_pcbpolicy(inp);
 318         if (error != 0) {
 319 #ifdef MAC
 320                 mac_inpcb_destroy(inp);
 321 #endif
 322                 goto out;
 323         }
 324 #endif /*IPSEC*/
 325 #ifdef INET6
 326         if (INP_SOCKAF(so) == AF_INET6) {
 327                 inp->inp_vflag |= INP_IPV6PROTO;
 328                 if (V_ip6_v6only)
 329                         inp->inp_flags |= IN6P_IPV6_V6ONLY;
 330         }
 331 #endif
 332         INP_WLOCK(inp);
 333         INP_LIST_WLOCK(pcbinfo);
 334         LIST_INSERT_HEAD(pcbinfo->ipi_listhead, inp, inp_list);
 335         pcbinfo->ipi_count++;
 336         so->so_pcb = (caddr_t)inp;
 337 #ifdef INET6
 338         if (V_ip6_auto_flowlabel)
 339                 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 340 #endif
 341         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 342         refcount_init(&inp->inp_refcount, 1);   /* Reference from inpcbinfo */
 343
 344         /*
 345          * Routes in inpcb's can cache L2 as well; they are guaranteed
 346          * to be cleaned up.
 347          */
 348         inp->inp_route.ro_flags = RT_LLE_CACHE;
 349         INP_LIST_WUNLOCK(pcbinfo);
 350 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
 351 out:
 352         if (error != 0) {
 353                 crfree(inp->inp_cred);
 354                 uma_zfree(pcbinfo->ipi_zone, inp);
 355         }
 356 #endif
 357         return (error);
 358 }
 359
 360 #ifdef INET
 361 int
 362 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 363 {
 364         int anonport, error;
 365
 366         INP_WLOCK_ASSERT(inp);
 367         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 368
 369         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 370                 return (EINVAL);
 371         anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
 372         error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
 373             &inp->inp_lport, cred);
 374         if (error)
 375                 return (error);
 376         if (in_pcbinshash(inp) != 0) {
 377                 inp->inp_laddr.s_addr = INADDR_ANY;
 378                 inp->inp_lport = 0;
 379                 return (EAGAIN);
 380         }
 381         if (anonport)
 382                 inp->inp_flags |= INP_ANONPORT;
 383         return (0);
 384 }
 385 #endif
 386
 387 /*
 388  * Select a local port (number) to use.
 389  */
 390 #if defined(INET) || defined(INET6)
 391 int
 392 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
 393     struct ucred *cred, int lookupflags)
 394 {
 395         struct inpcbinfo *pcbinfo;
 396         struct inpcb *tmpinp;
 397         unsigned short *lastport;
 398         int count, dorandom, error;
 399         u_short aux, first, last, lport;
 400 #ifdef INET
 401         struct in_addr laddr;
 402 #endif
 403
 404         pcbinfo = inp->inp_pcbinfo;
 405
 406         /*
 407          * Because no actual state changes occur here, a global write lock on
 408          * the pcbinfo isn't required.
 409          */
 410         INP_LOCK_ASSERT(inp);
 411         INP_HASH_LOCK_ASSERT(pcbinfo);
 412
 413         if (inp->inp_flags & INP_HIGHPORT) {
 414                 first = V_ipport_hifirstauto;   /* sysctl */
 415                 last  = V_ipport_hilastauto;
 416                 lastport = &pcbinfo->ipi_lasthi;
 417         } else if (inp->inp_flags & INP_LOWPORT) {
 418                 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0);
 419                 if (error)
 420                         return (error);
 421                 first = V_ipport_lowfirstauto;  /* 1023 */
 422                 last  = V_ipport_lowlastauto;   /* 600 */
 423                 lastport = &pcbinfo->ipi_lastlow;
 424         } else {
 425                 first = V_ipport_firstauto;     /* sysctl */
 426                 last  = V_ipport_lastauto;
 427                 lastport = &pcbinfo->ipi_lastport;
 428         }
 429         /*
 430          * For UDP(-Lite), use random port allocation as long as the user
 431          * allows it.  For TCP (and as of yet unknown) connections,
 432          * use random port allocation only if the user allows it AND
 433          * ipport_tick() allows it.
 434          */
 435         if (V_ipport_randomized &&
 436                 (!V_ipport_stoprandom || pcbinfo == &V_udbinfo ||
 437                 pcbinfo == &V_ulitecbinfo))
 438                 dorandom = 1;
 439         else
 440                 dorandom = 0;
 441         /*
 442          * It makes no sense to do random port allocation if
 443          * we have the only port available.
 444          */
 445         if (first == last)
 446                 dorandom = 0;
 447         /* Make sure to not include UDP(-Lite) packets in the count. */
 448         if (pcbinfo != &V_udbinfo || pcbinfo != &V_ulitecbinfo)
 449                 V_ipport_tcpallocs++;
 450         /*
 451          * Instead of having two loops further down counting up or down
 452          * make sure that first is always <= last and go with only one
 453          * code path implementing all logic.
 454          */
 455         if (first > last) {
 456                 aux = first;
 457                 first = last;
 458                 last = aux;
 459         }
 460
 461 #ifdef INET
 462         /* Make the compiler happy. */
 463         laddr.s_addr = 0;
 464         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
 465                 KASSERT(laddrp != NULL, ("%s: laddrp NULL for v4 inp %p",
 466                     __func__, inp));
 467                 laddr = *laddrp;
 468         }
 469 #endif
 470         tmpinp = NULL;  /* Make compiler happy. */
 471         lport = *lportp;
 472
 473         if (dorandom)
 474                 *lastport = first + (arc4random() % (last - first));
 475
 476         count = last - first;
 477
 478         do {
 479                 if (count-- < 0)        /* completely used? */
 480                         return (EADDRNOTAVAIL);
 481                 ++*lastport;
 482                 if (*lastport < first || *lastport > last)
 483                         *lastport = first;
 484                 lport = htons(*lastport);
 485
 486 #ifdef INET6
 487                 if ((inp->inp_vflag & INP_IPV6) != 0)
 488                         tmpinp = in6_pcblookup_local(pcbinfo,
 489                             &inp->in6p_laddr, lport, lookupflags, cred);
 490 #endif
 491 #if defined(INET) && defined(INET6)
 492                 else
 493 #endif
 494 #ifdef INET
 495                         tmpinp = in_pcblookup_local(pcbinfo, laddr,
 496                             lport, lookupflags, cred);
 497 #endif
 498         } while (tmpinp != NULL);
 499
 500 #ifdef INET
 501         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4)
 502                 laddrp->s_addr = laddr.s_addr;
 503 #endif
 504         *lportp = lport;
 505
 506         return (0);
 507 }
 508
 509 /*
 510  * Return cached socket options.
 511  */
 512 short
 513 inp_so_options(const struct inpcb *inp)
 514 {
 515    short so_options;
 516
 517    so_options = 0;
 518
 519    if ((inp->inp_flags2 & INP_REUSEPORT) != 0)
 520            so_options |= SO_REUSEPORT;
 521    if ((inp->inp_flags2 & INP_REUSEADDR) != 0)
 522            so_options |= SO_REUSEADDR;
 523    return (so_options);
 524 }
 525 #endif /* INET || INET6 */
 526
 527 /*
 528  * Check if a new BINDMULTI socket is allowed to be created.
 529  *
 530  * ni points to the new inp.
 531  * oi points to the exisitng inp.
 532  *
 533  * This checks whether the existing inp also has BINDMULTI and
 534  * whether the credentials match.
 535  */
 536 int
 537 in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
 538 {
 539         /* Check permissions match */
 540         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 541             (ni->inp_cred->cr_uid !=
 542             oi->inp_cred->cr_uid))
 543                 return (0);
 544
 545         /* Check the existing inp has BINDMULTI set */
 546         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 547             ((oi->inp_flags2 & INP_BINDMULTI) == 0))
 548                 return (0);
 549
 550         /*
 551          * We're okay - either INP_BINDMULTI isn't set on ni, or
 552          * it is and it matches the checks.
 553          */
 554         return (1);
 555 }
 556
 557 #ifdef INET
 558 /*
 559  * Set up a bind operation on a PCB, performing port allocation
 560  * as required, but do not actually modify the PCB. Callers can
 561  * either complete the bind by setting inp_laddr/inp_lport and
 562  * calling in_pcbinshash(), or they can just use the resulting
 563  * port and address to authorise the sending of a once-off packet.
 564  *
 565  * On error, the values of *laddrp and *lportp are not changed.
 566  */
 567 int
 568 in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
 569     u_short *lportp, struct ucred *cred)
 570 {
 571         struct socket *so = inp->inp_socket;
 572         struct sockaddr_in *sin;
 573         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 574         struct in_addr laddr;
 575         u_short lport = 0;
 576         int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
 577         int error;
 578
 579         /*
 580          * No state changes, so read locks are sufficient here.
 581          */
 582         INP_LOCK_ASSERT(inp);
 583         INP_HASH_LOCK_ASSERT(pcbinfo);
 584
 585         if (TAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */
 586                 return (EADDRNOTAVAIL);
 587         laddr.s_addr = *laddrp;
 588         if (nam != NULL && laddr.s_addr != INADDR_ANY)
 589                 return (EINVAL);
 590         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) == 0)
 591                 lookupflags = INPLOOKUP_WILDCARD;
 592         if (nam == NULL) {
 593                 if ((error = prison_local_ip4(cred, &laddr)) != 0)
 594                         return (error);
 595         } else {
 596                 sin = (struct sockaddr_in *)nam;
 597                 if (nam->sa_len != sizeof (*sin))
 598                         return (EINVAL);
 599 #ifdef notdef
 600                 /*
 601                  * We should check the family, but old programs
 602                  * incorrectly fail to initialize it.
 603                  */
 604                 if (sin->sin_family != AF_INET)
 605                         return (EAFNOSUPPORT);
 606 #endif
 607                 error = prison_local_ip4(cred, &sin->sin_addr);
 608                 if (error)
 609                         return (error);
 610                 if (sin->sin_port != *lportp) {
 611                         /* Don't allow the port to change. */
 612                         if (*lportp != 0)
 613                                 return (EINVAL);
 614                         lport = sin->sin_port;
 615                 }
 616                 /* NB: lport is left as 0 if the port isn't being changed. */
 617                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 618                         /*
 619                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 620                          * allow complete duplication of binding if
 621                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 622                          * and a multicast address is bound on both
 623                          * new and duplicated sockets.
 624                          */
 625                         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
 626                                 reuseport = SO_REUSEADDR|SO_REUSEPORT;
 627                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 628                         sin->sin_port = 0;              /* yech... */
 629                         bzero(&sin->sin_zero, sizeof(sin->sin_zero));
 630                         /*
 631                          * Is the address a local IP address?
 632                          * If INP_BINDANY is set, then the socket may be bound
 633                          * to any endpoint address, local or not.
 634                          */
 635                         if ((inp->inp_flags & INP_BINDANY) == 0 &&
 636                             ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
 637                                 return (EADDRNOTAVAIL);
 638                 }
 639                 laddr = sin->sin_addr;
 640                 if (lport) {
 641                         struct inpcb *t;
 642                         struct tcptw *tw;
 643
 644                         /* GROSS */
 645                         if (ntohs(lport) <= V_ipport_reservedhigh &&
 646                             ntohs(lport) >= V_ipport_reservedlow &&
 647                             priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT,
 648                             0))
 649                                 return (EACCES);
 650                         if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 651                             priv_check_cred(inp->inp_cred,
 652                             PRIV_NETINET_REUSEPORT, 0) != 0) {
 653                                 t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 654                                     lport, INPLOOKUP_WILDCARD, cred);
 655         /*
 656          * XXX
 657          * This entire block sorely needs a rewrite.
 658          */
 659                                 if (t &&
 660                                     ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 661                                     ((t->inp_flags & INP_TIMEWAIT) == 0) &&
 662                                     (so->so_type != SOCK_STREAM ||
 663                                      ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
 664                                     (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
 665                                      ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
 666                                      (t->inp_flags2 & INP_REUSEPORT) == 0) &&
 667                                     (inp->inp_cred->cr_uid !=
 668                                      t->inp_cred->cr_uid))
 669                                         return (EADDRINUSE);
 670
 671                                 /*
 672                                  * If the socket is a BINDMULTI socket, then
 673                                  * the credentials need to match and the
 674                                  * original socket also has to have been bound
 675                                  * with BINDMULTI.
 676                                  */
 677                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 678                                         return (EADDRINUSE);
 679                         }
 680                         t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 681                             lport, lookupflags, cred);
 682                         if (t && (t->inp_flags & INP_TIMEWAIT)) {
 683                                 /*
 684                                  * XXXRW: If an incpb has had its timewait
 685                                  * state recycled, we treat the address as
 686                                  * being in use (for now).  This is better
 687                                  * than a panic, but not desirable.
 688                                  */
 689                                 tw = intotw(t);
 690                                 if (tw == NULL ||
 691                                     (reuseport & tw->tw_so_options) == 0)
 692                                         return (EADDRINUSE);
 693                         } else if (t &&
 694                             ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 695                             (reuseport & inp_so_options(t)) == 0) {
 696 #ifdef INET6
 697                                 if (ntohl(sin->sin_addr.s_addr) !=
 698                                     INADDR_ANY ||
 699                                     ntohl(t->inp_laddr.s_addr) !=
 700                                     INADDR_ANY ||
 701                                     (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
 702                                     (t->inp_vflag & INP_IPV6PROTO) == 0)
 703 #endif
 704                                 return (EADDRINUSE);
 705                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 706                                         return (EADDRINUSE);
 707                         }
 708                 }
 709         }
 710         if (*lportp != 0)
 711                 lport = *lportp;
 712         if (lport == 0) {
 713                 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
 714                 if (error != 0)
 715                         return (error);
 716
 717         }
 718         *laddrp = laddr.s_addr;
 719         *lportp = lport;
 720         return (0);
 721 }
 722
 723 /*
 724  * Connect from a socket to a specified address.
 725  * Both address and port must be specified in argument sin.
 726  * If don't have a local address for this socket yet,
 727  * then pick one.
 728  */
 729 int
 730 in_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
 731     struct ucred *cred, struct mbuf *m)
 732 {
 733         u_short lport, fport;
 734         in_addr_t laddr, faddr;
 735         int anonport, error;
 736
 737         INP_WLOCK_ASSERT(inp);
 738         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 739
 740         lport = inp->inp_lport;
 741         laddr = inp->inp_laddr.s_addr;
 742         anonport = (lport == 0);
 743         error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport,
 744             NULL, cred);
 745         if (error)
 746                 return (error);
 747
 748         /* Do the initial binding of the local address if required. */
 749         if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
 750                 inp->inp_lport = lport;
 751                 inp->inp_laddr.s_addr = laddr;
 752                 if (in_pcbinshash(inp) != 0) {
 753                         inp->inp_laddr.s_addr = INADDR_ANY;
 754                         inp->inp_lport = 0;
 755                         return (EAGAIN);
 756                 }
 757         }
 758
 759         /* Commit the remaining changes. */
 760         inp->inp_lport = lport;
 761         inp->inp_laddr.s_addr = laddr;
 762         inp->inp_faddr.s_addr = faddr;
 763         inp->inp_fport = fport;
 764         in_pcbrehash_mbuf(inp, m);
 765
 766         if (anonport)
 767                 inp->inp_flags |= INP_ANONPORT;
 768         return (0);
 769 }
 770
 771 int
 772 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 773 {
 774
 775         return (in_pcbconnect_mbuf(inp, nam, cred, NULL));
 776 }
 777
 778 /*
 779  * Do proper source address selection on an unbound socket in case
 780  * of connect. Take jails into account as well.
 781  */
 782 int
 783 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
 784     struct ucred *cred)
 785 {
 786         struct ifaddr *ifa;
 787         struct sockaddr *sa;
 788         struct sockaddr_in *sin;
 789         struct route sro;
 790         int error;
 791
 792         KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
 793
 794         /*
 795          * Bypass source address selection and use the primary jail IP
 796          * if requested.
 797          */
 798         if (cred != NULL && !prison_saddrsel_ip4(cred, laddr))
 799                 return (0);
 800
 801         error = 0;
 802         bzero(&sro, sizeof(sro));
 803
 804         sin = (struct sockaddr_in *)&sro.ro_dst;
 805         sin->sin_family = AF_INET;
 806         sin->sin_len = sizeof(struct sockaddr_in);
 807         sin->sin_addr.s_addr = faddr->s_addr;
 808
 809         /*
 810          * If route is known our src addr is taken from the i/f,
 811          * else punt.
 812          *
 813          * Find out route to destination.
 814          */
 815         if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
 816                 in_rtalloc_ign(&sro, 0, inp->inp_inc.inc_fibnum);
 817
 818         /*
 819          * If we found a route, use the address corresponding to
 820          * the outgoing interface.
 821          *
 822          * Otherwise assume faddr is reachable on a directly connected
 823          * network and try to find a corresponding interface to take
 824          * the source address from.
 825          */
 826         if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) {
 827                 struct in_ifaddr *ia;
 828                 struct ifnet *ifp;
 829
 830                 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
 831                                         inp->inp_socket->so_fibnum));
 832                 if (ia == NULL)
 833                         ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
 834                                                 inp->inp_socket->so_fibnum));
 835                 if (ia == NULL) {
 836                         error = ENETUNREACH;
 837                         goto done;
 838                 }
 839
 840                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 841                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 842                         ifa_free(&ia->ia_ifa);
 843                         goto done;
 844                 }
 845
 846                 ifp = ia->ia_ifp;
 847                 ifa_free(&ia->ia_ifa);
 848                 ia = NULL;
 849                 IF_ADDR_RLOCK(ifp);
 850                 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 851
 852                         sa = ifa->ifa_addr;
 853                         if (sa->sa_family != AF_INET)
 854                                 continue;
 855                         sin = (struct sockaddr_in *)sa;
 856                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 857                                 ia = (struct in_ifaddr *)ifa;
 858                                 break;
 859                         }
 860                 }
 861                 if (ia != NULL) {
 862                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 863                         IF_ADDR_RUNLOCK(ifp);
 864                         goto done;
 865                 }
 866                 IF_ADDR_RUNLOCK(ifp);
 867
 868                 /* 3. As a last resort return the 'default' jail address. */
 869                 error = prison_get_ip4(cred, laddr);
 870                 goto done;
 871         }
 872
 873         /*
 874          * If the outgoing interface on the route found is not
 875          * a loopback interface, use the address from that interface.
 876          * In case of jails do those three steps:
 877          * 1. check if the interface address belongs to the jail. If so use it.
 878          * 2. check if we have any address on the outgoing interface
 879          *    belonging to this jail. If so use it.
 880          * 3. as a last resort return the 'default' jail address.
 881          */
 882         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) == 0) {
 883                 struct in_ifaddr *ia;
 884                 struct ifnet *ifp;
 885
 886                 /* If not jailed, use the default returned. */
 887                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 888                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 889                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 890                         goto done;
 891                 }
 892
 893                 /* Jailed. */
 894                 /* 1. Check if the iface address belongs to the jail. */
 895                 sin = (struct sockaddr_in *)sro.ro_rt->rt_ifa->ifa_addr;
 896                 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 897                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 898                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 899                         goto done;
 900                 }
 901
 902                 /*
 903                  * 2. Check if we have any address on the outgoing interface
 904                  *    belonging to this jail.
 905                  */
 906                 ia = NULL;
 907                 ifp = sro.ro_rt->rt_ifp;
 908                 IF_ADDR_RLOCK(ifp);
 909                 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 910                         sa = ifa->ifa_addr;
 911                         if (sa->sa_family != AF_INET)
 912                                 continue;
 913                         sin = (struct sockaddr_in *)sa;
 914                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 915                                 ia = (struct in_ifaddr *)ifa;
 916                                 break;
 917                         }
 918                 }
 919                 if (ia != NULL) {
 920                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 921                         IF_ADDR_RUNLOCK(ifp);
 922                         goto done;
 923                 }
 924                 IF_ADDR_RUNLOCK(ifp);
 925
 926                 /* 3. As a last resort return the 'default' jail address. */
 927                 error = prison_get_ip4(cred, laddr);
 928                 goto done;
 929         }
 930
 931         /*
 932          * The outgoing interface is marked with 'loopback net', so a route
 933          * to ourselves is here.
 934          * Try to find the interface of the destination address and then
 935          * take the address from there. That interface is not necessarily
 936          * a loopback interface.
 937          * In case of jails, check that it is an address of the jail
 938          * and if we cannot find, fall back to the 'default' jail address.
 939          */
 940         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
 941                 struct sockaddr_in sain;
 942                 struct in_ifaddr *ia;
 943
 944                 bzero(&sain, sizeof(struct sockaddr_in));
 945                 sain.sin_family = AF_INET;
 946                 sain.sin_len = sizeof(struct sockaddr_in);
 947                 sain.sin_addr.s_addr = faddr->s_addr;
 948
 949                 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&sain),
 950                                         inp->inp_socket->so_fibnum));
 951                 if (ia == NULL)
 952                         ia = ifatoia(ifa_ifwithnet(sintosa(&sain), 0,
 953                                                 inp->inp_socket->so_fibnum));
 954                 if (ia == NULL)
 955                         ia = ifatoia(ifa_ifwithaddr(sintosa(&sain)));
 956
 957                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 958                         if (ia == NULL) {
 959                                 error = ENETUNREACH;
 960                                 goto done;
 961                         }
 962                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 963                         ifa_free(&ia->ia_ifa);
 964                         goto done;
 965                 }
 966
 967                 /* Jailed. */
 968                 if (ia != NULL) {
 969                         struct ifnet *ifp;
 970
 971                         ifp = ia->ia_ifp;
 972                         ifa_free(&ia->ia_ifa);
 973                         ia = NULL;
 974                         IF_ADDR_RLOCK(ifp);
 975                         TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 976
 977                                 sa = ifa->ifa_addr;
 978                                 if (sa->sa_family != AF_INET)
 979                                         continue;
 980                                 sin = (struct sockaddr_in *)sa;
 981                                 if (prison_check_ip4(cred,
 982                                     &sin->sin_addr) == 0) {
 983                                         ia = (struct in_ifaddr *)ifa;
 984                                         break;
 985                                 }
 986                         }
 987                         if (ia != NULL) {
 988                                 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 989                                 IF_ADDR_RUNLOCK(ifp);
 990                                 goto done;
 991                         }
 992                         IF_ADDR_RUNLOCK(ifp);
 993                 }
 994
 995                 /* 3. As a last resort return the 'default' jail address. */
 996                 error = prison_get_ip4(cred, laddr);
 997                 goto done;
 998         }
 999
1000 done:
1001         if (sro.ro_rt != NULL)
1002                 RTFREE(sro.ro_rt);
1003         return (error);
1004 }
1005
1006 /*
1007  * Set up for a connect from a socket to the specified address.
1008  * On entry, *laddrp and *lportp should contain the current local
1009  * address and port for the PCB; these are updated to the values
1010  * that should be placed in inp_laddr and inp_lport to complete
1011  * the connect.
1012  *
1013  * On success, *faddrp and *fportp will be set to the remote address
1014  * and port. These are not updated in the error case.
1015  *
1016  * If the operation fails because the connection already exists,
1017  * *oinpp will be set to the PCB of that connection so that the
1018  * caller can decide to override it. In all other cases, *oinpp
1019  * is set to NULL.
1020  */
1021 int
1022 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
1023     in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1024     struct inpcb **oinpp, struct ucred *cred)
1025 {
1026         struct rm_priotracker in_ifa_tracker;
1027         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1028         struct in_ifaddr *ia;
1029         struct inpcb *oinp;
1030         struct in_addr laddr, faddr;
1031         u_short lport, fport;
1032         int error;
1033
1034         /*
1035          * Because a global state change doesn't actually occur here, a read
1036          * lock is sufficient.
1037          */
1038         INP_LOCK_ASSERT(inp);
1039         INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1040
1041         if (oinpp != NULL)
1042                 *oinpp = NULL;
1043         if (nam->sa_len != sizeof (*sin))
1044                 return (EINVAL);
1045         if (sin->sin_family != AF_INET)
1046                 return (EAFNOSUPPORT);
1047         if (sin->sin_port == 0)
1048                 return (EADDRNOTAVAIL);
1049         laddr.s_addr = *laddrp;
1050         lport = *lportp;
1051         faddr = sin->sin_addr;
1052         fport = sin->sin_port;
1053
1054         if (!TAILQ_EMPTY(&V_in_ifaddrhead)) {
1055                 /*
1056                  * If the destination address is INADDR_ANY,
1057                  * use the primary local address.
1058                  * If the supplied address is INADDR_BROADCAST,
1059                  * and the primary interface supports broadcast,
1060                  * choose the broadcast address for that interface.
1061                  */
1062                 if (faddr.s_addr == INADDR_ANY) {
1063                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1064                         faddr =
1065                             IA_SIN(TAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1066                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1067                         if (cred != NULL &&
1068                             (error = prison_get_ip4(cred, &faddr)) != 0)
1069                                 return (error);
1070                 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1071                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1072                         if (TAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1073                             IFF_BROADCAST)
1074                                 faddr = satosin(&TAILQ_FIRST(
1075                                     &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1076                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1077                 }
1078         }
1079         if (laddr.s_addr == INADDR_ANY) {
1080                 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1081                 /*
1082                  * If the destination address is multicast and an outgoing
1083                  * interface has been set as a multicast option, prefer the
1084                  * address of that interface as our source address.
1085                  */
1086                 if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1087                     inp->inp_moptions != NULL) {
1088                         struct ip_moptions *imo;
1089                         struct ifnet *ifp;
1090
1091                         imo = inp->inp_moptions;
1092                         if (imo->imo_multicast_ifp != NULL) {
1093                                 ifp = imo->imo_multicast_ifp;
1094                                 IN_IFADDR_RLOCK(&in_ifa_tracker);
1095                                 TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1096                                         if ((ia->ia_ifp == ifp) &&
1097                                             (cred == NULL ||
1098                                             prison_check_ip4(cred,
1099                                             &ia->ia_addr.sin_addr) == 0))
1100                                                 break;
1101                                 }
1102                                 if (ia == NULL)
1103                                         error = EADDRNOTAVAIL;
1104                                 else {
1105                                         laddr = ia->ia_addr.sin_addr;
1106                                         error = 0;
1107                                 }
1108                                 IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1109                         }
1110                 }
1111                 if (error)
1112                         return (error);
1113         }
1114         oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, fport,
1115             laddr, lport, 0, NULL);
1116         if (oinp != NULL) {
1117                 if (oinpp != NULL)
1118                         *oinpp = oinp;
1119                 return (EADDRINUSE);
1120         }
1121         if (lport == 0) {
1122                 error = in_pcbbind_setup(inp, NULL, &laddr.s_addr, &lport,
1123                     cred);
1124                 if (error)
1125                         return (error);
1126         }
1127         *laddrp = laddr.s_addr;
1128         *lportp = lport;
1129         *faddrp = faddr.s_addr;
1130         *fportp = fport;
1131         return (0);
1132 }
1133
1134 void
1135 in_pcbdisconnect(struct inpcb *inp)
1136 {
1137
1138         INP_WLOCK_ASSERT(inp);
1139         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1140
1141         inp->inp_faddr.s_addr = INADDR_ANY;
1142         inp->inp_fport = 0;
1143         in_pcbrehash(inp);
1144 }
1145 #endif /* INET */
1146
1147 /*
1148  * in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
1149  * For most protocols, this will be invoked immediately prior to calling
1150  * in_pcbfree().  However, with TCP the inpcb may significantly outlive the
1151  * socket, in which case in_pcbfree() is deferred.
1152  */
1153 void
1154 in_pcbdetach(struct inpcb *inp)
1155 {
1156
1157         KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1158
1159 #ifdef RATELIMIT
1160         if (inp->inp_snd_tag != NULL)
1161                 in_pcbdetach_txrtlmt(inp);
1162 #endif
1163         inp->inp_socket->so_pcb = NULL;
1164         inp->inp_socket = NULL;
1165 }
1166
1167 /*
1168  * in_pcbref() bumps the reference count on an inpcb in order to maintain
1169  * stability of an inpcb pointer despite the inpcb lock being released.  This
1170  * is used in TCP when the inpcbinfo lock needs to be acquired or upgraded,
1171  * but where the inpcb lock may already held, or when acquiring a reference
1172  * via a pcbgroup.
1173  *
1174  * in_pcbref() should be used only to provide brief memory stability, and
1175  * must always be followed by a call to INP_WLOCK() and in_pcbrele() to
1176  * garbage collect the inpcb if it has been in_pcbfree()'d from another
1177  * context.  Until in_pcbrele() has returned that the inpcb is still valid,
1178  * lock and rele are the *only* safe operations that may be performed on the
1179  * inpcb.
1180  *
1181  * While the inpcb will not be freed, releasing the inpcb lock means that the
1182  * connection's state may change, so the caller should be careful to
1183  * revalidate any cached state on reacquiring the lock.  Drop the reference
1184  * using in_pcbrele().
1185  */
1186 void
1187 in_pcbref(struct inpcb *inp)
1188 {
1189
1190         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1191
1192         refcount_acquire(&inp->inp_refcount);
1193 }
1194
1195 /*
1196  * Drop a refcount on an inpcb elevated using in_pcbref(); because a call to
1197  * in_pcbfree() may have been made between in_pcbref() and in_pcbrele(), we
1198  * return a flag indicating whether or not the inpcb remains valid.  If it is
1199  * valid, we return with the inpcb lock held.
1200  *
1201  * Notice that, unlike in_pcbref(), the inpcb lock must be held to drop a
1202  * reference on an inpcb.  Historically more work was done here (actually, in
1203  * in_pcbfree_internal()) but has been moved to in_pcbfree() to avoid the
1204  * need for the pcbinfo lock in in_pcbrele().  Deferring the free is entirely
1205  * about memory stability (and continued use of the write lock).
1206  */
1207 int
1208 in_pcbrele_rlocked(struct inpcb *inp)
1209 {
1210         struct inpcbinfo *pcbinfo;
1211
1212         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1213
1214         INP_RLOCK_ASSERT(inp);
1215
1216         if (refcount_release(&inp->inp_refcount) == 0) {
1217                 /*
1218                  * If the inpcb has been freed, let the caller know, even if
1219                  * this isn't the last reference.
1220                  */
1221                 if (inp->inp_flags2 & INP_FREED) {
1222                         INP_RUNLOCK(inp);
1223                         return (1);
1224                 }
1225                 return (0);
1226         }
1227
1228         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1229
1230         INP_RUNLOCK(inp);
1231         pcbinfo = inp->inp_pcbinfo;
1232         uma_zfree(pcbinfo->ipi_zone, inp);
1233         return (1);
1234 }
1235
1236 int
1237 in_pcbrele_wlocked(struct inpcb *inp)
1238 {
1239         struct inpcbinfo *pcbinfo;
1240
1241         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1242
1243         INP_WLOCK_ASSERT(inp);
1244
1245         if (refcount_release(&inp->inp_refcount) == 0) {
1246                 /*
1247                  * If the inpcb has been freed, let the caller know, even if
1248                  * this isn't the last reference.
1249                  */
1250                 if (inp->inp_flags2 & INP_FREED) {
1251                         INP_WUNLOCK(inp);
1252                         return (1);
1253                 }
1254                 return (0);
1255         }
1256
1257         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1258
1259         INP_WUNLOCK(inp);
1260         pcbinfo = inp->inp_pcbinfo;
1261         uma_zfree(pcbinfo->ipi_zone, inp);
1262         return (1);
1263 }
1264
1265 /*
1266  * Temporary wrapper.
1267  */
1268 int
1269 in_pcbrele(struct inpcb *inp)
1270 {
1271
1272         return (in_pcbrele_wlocked(inp));
1273 }
1274
1275 /*
1276  * Unconditionally schedule an inpcb to be freed by decrementing its
1277  * reference count, which should occur only after the inpcb has been detached
1278  * from its socket.  If another thread holds a temporary reference (acquired
1279  * using in_pcbref()) then the free is deferred until that reference is
1280  * released using in_pcbrele(), but the inpcb is still unlocked.  Almost all
1281  * work, including removal from global lists, is done in this context, where
1282  * the pcbinfo lock is held.
1283  */
1284 void
1285 in_pcbfree(struct inpcb *inp)
1286 {
1287         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1288
1289         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1290
1291 #ifdef INVARIANTS
1292         if (pcbinfo == &V_tcbinfo) {
1293                 INP_INFO_LOCK_ASSERT(pcbinfo);
1294         } else {
1295                 INP_INFO_WLOCK_ASSERT(pcbinfo);
1296         }
1297 #endif
1298         INP_WLOCK_ASSERT(inp);
1299
1300         /* XXXRW: Do as much as possible here. */
1301 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1302         if (inp->inp_sp != NULL)
1303                 ipsec_delete_pcbpolicy(inp);
1304 #endif
1305         INP_LIST_WLOCK(pcbinfo);
1306         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1307         in_pcbremlists(inp);
1308         INP_LIST_WUNLOCK(pcbinfo);
1309 #ifdef INET6
1310         if (inp->inp_vflag & INP_IPV6PROTO) {
1311                 ip6_freepcbopts(inp->in6p_outputopts);
1312                 if (inp->in6p_moptions != NULL)
1313                         ip6_freemoptions(inp->in6p_moptions);
1314         }
1315 #endif
1316         if (inp->inp_options)
1317                 (void)m_free(inp->inp_options);
1318 #ifdef INET
1319         if (inp->inp_moptions != NULL)
1320                 inp_freemoptions(inp->inp_moptions);
1321 #endif
1322         RO_RTFREE(&inp->inp_route);
1323         if (inp->inp_route.ro_lle)
1324                 LLE_FREE(inp->inp_route.ro_lle);        /* zeros ro_lle */
1325
1326         inp->inp_vflag = 0;
1327         inp->inp_flags2 |= INP_FREED;
1328         crfree(inp->inp_cred);
1329 #ifdef MAC
1330         mac_inpcb_destroy(inp);
1331 #endif
1332         if (!in_pcbrele_wlocked(inp))
1333                 INP_WUNLOCK(inp);
1334 }
1335
1336 /*
1337  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1338  * port reservation, and preventing it from being returned by inpcb lookups.
1339  *
1340  * It is used by TCP to mark an inpcb as unused and avoid future packet
1341  * delivery or event notification when a socket remains open but TCP has
1342  * closed.  This might occur as a result of a shutdown()-initiated TCP close
1343  * or a RST on the wire, and allows the port binding to be reused while still
1344  * maintaining the invariant that so_pcb always points to a valid inpcb until
1345  * in_pcbdetach().
1346  *
1347  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1348  * in_pcbnotifyall() and in_pcbpurgeif0()?
1349  */
1350 void
1351 in_pcbdrop(struct inpcb *inp)
1352 {
1353
1354         INP_WLOCK_ASSERT(inp);
1355
1356         /*
1357          * XXXRW: Possibly we should protect the setting of INP_DROPPED with
1358          * the hash lock...?
1359          */
1360         inp->inp_flags |= INP_DROPPED;
1361         if (inp->inp_flags & INP_INHASHLIST) {
1362                 struct inpcbport *phd = inp->inp_phd;
1363
1364                 INP_HASH_WLOCK(inp->inp_pcbinfo);
1365                 LIST_REMOVE(inp, inp_hash);
1366                 LIST_REMOVE(inp, inp_portlist);
1367                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
1368                         LIST_REMOVE(phd, phd_hash);
1369                         free(phd, M_PCB);
1370                 }
1371                 INP_HASH_WUNLOCK(inp->inp_pcbinfo);
1372                 inp->inp_flags &= ~INP_INHASHLIST;
1373 #ifdef PCBGROUP
1374                 in_pcbgroup_remove(inp);
1375 #endif
1376         }
1377 }
1378
1379 #ifdef INET
1380 /*
1381  * Common routines to return the socket addresses associated with inpcbs.
1382  */
1383 struct sockaddr *
1384 in_sockaddr(in_port_t port, struct in_addr *addr_p)
1385 {
1386         struct sockaddr_in *sin;
1387
1388         sin = malloc(sizeof *sin, M_SONAME,
1389                 M_WAITOK | M_ZERO);
1390         sin->sin_family = AF_INET;
1391         sin->sin_len = sizeof(*sin);
1392         sin->sin_addr = *addr_p;
1393         sin->sin_port = port;
1394
1395         return (struct sockaddr *)sin;
1396 }
1397
1398 int
1399 in_getsockaddr(struct socket *so, struct sockaddr **nam)
1400 {
1401         struct inpcb *inp;
1402         struct in_addr addr;
1403         in_port_t port;
1404
1405         inp = sotoinpcb(so);
1406         KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1407
1408         INP_RLOCK(inp);
1409         port = inp->inp_lport;
1410         addr = inp->inp_laddr;
1411         INP_RUNLOCK(inp);
1412
1413         *nam = in_sockaddr(port, &addr);
1414         return 0;
1415 }
1416
1417 int
1418 in_getpeeraddr(struct socket *so, struct sockaddr **nam)
1419 {
1420         struct inpcb *inp;
1421         struct in_addr addr;
1422         in_port_t port;
1423
1424         inp = sotoinpcb(so);
1425         KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1426
1427         INP_RLOCK(inp);
1428         port = inp->inp_fport;
1429         addr = inp->inp_faddr;
1430         INP_RUNLOCK(inp);
1431
1432         *nam = in_sockaddr(port, &addr);
1433         return 0;
1434 }
1435
1436 void
1437 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
1438     struct inpcb *(*notify)(struct inpcb *, int))
1439 {
1440         struct inpcb *inp, *inp_temp;
1441
1442         INP_INFO_WLOCK(pcbinfo);
1443         LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
1444                 INP_WLOCK(inp);
1445 #ifdef INET6
1446                 if ((inp->inp_vflag & INP_IPV4) == 0) {
1447                         INP_WUNLOCK(inp);
1448                         continue;
1449                 }
1450 #endif
1451                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1452                     inp->inp_socket == NULL) {
1453                         INP_WUNLOCK(inp);
1454                         continue;
1455                 }
1456                 if ((*notify)(inp, errno))
1457                         INP_WUNLOCK(inp);
1458         }
1459         INP_INFO_WUNLOCK(pcbinfo);
1460 }
1461
1462 void
1463 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1464 {
1465         struct inpcb *inp;
1466         struct ip_moptions *imo;
1467         int i, gap;
1468
1469         INP_INFO_WLOCK(pcbinfo);
1470         LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1471                 INP_WLOCK(inp);
1472                 imo = inp->inp_moptions;
1473                 if ((inp->inp_vflag & INP_IPV4) &&
1474                     imo != NULL) {
1475                         /*
1476                          * Unselect the outgoing interface if it is being
1477                          * detached.
1478                          */
1479                         if (imo->imo_multicast_ifp == ifp)
1480                                 imo->imo_multicast_ifp = NULL;
1481
1482                         /*
1483                          * Drop multicast group membership if we joined
1484                          * through the interface being detached.
1485                          */
1486                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1487                             i++) {
1488                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1489                                         in_delmulti(imo->imo_membership[i]);
1490                                         gap++;
1491                                 } else if (gap != 0)
1492                                         imo->imo_membership[i - gap] =
1493                                             imo->imo_membership[i];
1494                         }
1495                         imo->imo_num_memberships -= gap;
1496                 }
1497                 INP_WUNLOCK(inp);
1498         }
1499         INP_INFO_WUNLOCK(pcbinfo);
1500 }
1501
1502 /*
1503  * Lookup a PCB based on the local address and port.  Caller must hold the
1504  * hash lock.  No inpcb locks or references are acquired.
1505  */
1506 #define INP_LOOKUP_MAPPED_PCB_COST      3
1507 struct inpcb *
1508 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1509     u_short lport, int lookupflags, struct ucred *cred)
1510 {
1511         struct inpcb *inp;
1512 #ifdef INET6
1513         int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1514 #else
1515         int matchwild = 3;
1516 #endif
1517         int wildcard;
1518
1519         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1520             ("%s: invalid lookup flags %d", __func__, lookupflags));
1521
1522         INP_HASH_LOCK_ASSERT(pcbinfo);
1523
1524         if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1525                 struct inpcbhead *head;
1526                 /*
1527                  * Look for an unconnected (wildcard foreign addr) PCB that
1528                  * matches the local address and port we're looking for.
1529                  */
1530                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1531                     0, pcbinfo->ipi_hashmask)];
1532                 LIST_FOREACH(inp, head, inp_hash) {
1533 #ifdef INET6
1534                         /* XXX inp locking */
1535                         if ((inp->inp_vflag & INP_IPV4) == 0)
1536                                 continue;
1537 #endif
1538                         if (inp->inp_faddr.s_addr == INADDR_ANY &&
1539                             inp->inp_laddr.s_addr == laddr.s_addr &&
1540                             inp->inp_lport == lport) {
1541                                 /*
1542                                  * Found?
1543                                  */
1544                                 if (cred == NULL ||
1545                                     prison_equal_ip4(cred->cr_prison,
1546                                         inp->inp_cred->cr_prison))
1547                                         return (inp);
1548                         }
1549                 }
1550                 /*
1551                  * Not found.
1552                  */
1553                 return (NULL);
1554         } else {
1555                 struct inpcbporthead *porthash;
1556                 struct inpcbport *phd;
1557                 struct inpcb *match = NULL;
1558                 /*
1559                  * Best fit PCB lookup.
1560                  *
1561                  * First see if this local port is in use by looking on the
1562                  * port hash list.
1563                  */
1564                 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
1565                     pcbinfo->ipi_porthashmask)];
1566                 LIST_FOREACH(phd, porthash, phd_hash) {
1567                         if (phd->phd_port == lport)
1568                                 break;
1569                 }
1570                 if (phd != NULL) {
1571                         /*
1572                          * Port is in use by one or more PCBs. Look for best
1573                          * fit.
1574                          */
1575                         LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1576                                 wildcard = 0;
1577                                 if (cred != NULL &&
1578                                     !prison_equal_ip4(inp->inp_cred->cr_prison,
1579                                         cred->cr_prison))
1580                                         continue;
1581 #ifdef INET6
1582                                 /* XXX inp locking */
1583                                 if ((inp->inp_vflag & INP_IPV4) == 0)
1584                                         continue;
1585                                 /*
1586                                  * We never select the PCB that has
1587                                  * INP_IPV6 flag and is bound to :: if
1588                                  * we have another PCB which is bound
1589                                  * to 0.0.0.0.  If a PCB has the
1590                                  * INP_IPV6 flag, then we set its cost
1591                                  * higher than IPv4 only PCBs.
1592                                  *
1593                                  * Note that the case only happens
1594                                  * when a socket is bound to ::, under
1595                                  * the condition that the use of the
1596                                  * mapped address is allowed.
1597                                  */
1598                                 if ((inp->inp_vflag & INP_IPV6) != 0)
1599                                         wildcard += INP_LOOKUP_MAPPED_PCB_COST;
1600 #endif
1601                                 if (inp->inp_faddr.s_addr != INADDR_ANY)
1602                                         wildcard++;
1603                                 if (inp->inp_laddr.s_addr != INADDR_ANY) {
1604                                         if (laddr.s_addr == INADDR_ANY)
1605                                                 wildcard++;
1606                                         else if (inp->inp_laddr.s_addr != laddr.s_addr)
1607                                                 continue;
1608                                 } else {
1609                                         if (laddr.s_addr != INADDR_ANY)
1610                                                 wildcard++;
1611                                 }
1612                                 if (wildcard < matchwild) {
1613                                         match = inp;
1614                                         matchwild = wildcard;
1615                                         if (matchwild == 0)
1616                                                 break;
1617                                 }
1618                         }
1619                 }
1620                 return (match);
1621         }
1622 }
1623 #undef INP_LOOKUP_MAPPED_PCB_COST
1624
1625 #ifdef PCBGROUP
1626 /*
1627  * Lookup PCB in hash list, using pcbgroup tables.
1628  */
1629 static struct inpcb *
1630 in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
1631     struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
1632     u_int lport_arg, int lookupflags, struct ifnet *ifp)
1633 {
1634         struct inpcbhead *head;
1635         struct inpcb *inp, *tmpinp;
1636         u_short fport = fport_arg, lport = lport_arg;
1637
1638         /*
1639          * First look for an exact match.
1640          */
1641         tmpinp = NULL;
1642         INP_GROUP_LOCK(pcbgroup);
1643         head = &pcbgroup->ipg_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
1644             pcbgroup->ipg_hashmask)];
1645         LIST_FOREACH(inp, head, inp_pcbgrouphash) {
1646 #ifdef INET6
1647                 /* XXX inp locking */
1648                 if ((inp->inp_vflag & INP_IPV4) == 0)
1649                         continue;
1650 #endif
1651                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
1652                     inp->inp_laddr.s_addr == laddr.s_addr &&
1653                     inp->inp_fport == fport &&
1654                     inp->inp_lport == lport) {
1655                         /*
1656                          * XXX We should be able to directly return
1657                          * the inp here, without any checks.
1658                          * Well unless both bound with SO_REUSEPORT?
1659                          */
1660                         if (prison_flag(inp->inp_cred, PR_IP4))
1661                                 goto found;
1662                         if (tmpinp == NULL)
1663                                 tmpinp = inp;
1664                 }
1665         }
1666         if (tmpinp != NULL) {
1667                 inp = tmpinp;
1668                 goto found;
1669         }
1670
1671 #ifdef  RSS
1672         /*
1673          * For incoming connections, we may wish to do a wildcard
1674          * match for an RSS-local socket.
1675          */
1676         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1677                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1678 #ifdef INET6
1679                 struct inpcb *local_wild_mapped = NULL;
1680 #endif
1681                 struct inpcb *jail_wild = NULL;
1682                 struct inpcbhead *head;
1683                 int injail;
1684
1685                 /*
1686                  * Order of socket selection - we always prefer jails.
1687                  *      1. jailed, non-wild.
1688                  *      2. jailed, wild.
1689                  *      3. non-jailed, non-wild.
1690                  *      4. non-jailed, wild.
1691                  */
1692
1693                 head = &pcbgroup->ipg_hashbase[INP_PCBHASH(INADDR_ANY,
1694                     lport, 0, pcbgroup->ipg_hashmask)];
1695                 LIST_FOREACH(inp, head, inp_pcbgrouphash) {
1696 #ifdef INET6
1697                         /* XXX inp locking */
1698                         if ((inp->inp_vflag & INP_IPV4) == 0)
1699                                 continue;
1700 #endif
1701                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1702                             inp->inp_lport != lport)
1703                                 continue;
1704
1705                         injail = prison_flag(inp->inp_cred, PR_IP4);
1706                         if (injail) {
1707                                 if (prison_check_ip4(inp->inp_cred,
1708                                     &laddr) != 0)
1709                                         continue;
1710                         } else {
1711                                 if (local_exact != NULL)
1712                                         continue;
1713                         }
1714
1715                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1716                                 if (injail)
1717                                         goto found;
1718                                 else
1719                                         local_exact = inp;
1720                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1721 #ifdef INET6
1722                                 /* XXX inp locking, NULL check */
1723                                 if (inp->inp_vflag & INP_IPV6PROTO)
1724                                         local_wild_mapped = inp;
1725                                 else
1726 #endif
1727                                         if (injail)
1728                                                 jail_wild = inp;
1729                                         else
1730                                                 local_wild = inp;
1731                         }
1732                 } /* LIST_FOREACH */
1733
1734                 inp = jail_wild;
1735                 if (inp == NULL)
1736                         inp = local_exact;
1737                 if (inp == NULL)
1738                         inp = local_wild;
1739 #ifdef INET6
1740                 if (inp == NULL)
1741                         inp = local_wild_mapped;
1742 #endif
1743                 if (inp != NULL)
1744                         goto found;
1745         }
1746 #endif
1747
1748         /*
1749          * Then look for a wildcard match, if requested.
1750          */
1751         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1752                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1753 #ifdef INET6
1754                 struct inpcb *local_wild_mapped = NULL;
1755 #endif
1756                 struct inpcb *jail_wild = NULL;
1757                 struct inpcbhead *head;
1758                 int injail;
1759
1760                 /*
1761                  * Order of socket selection - we always prefer jails.
1762                  *      1. jailed, non-wild.
1763                  *      2. jailed, wild.
1764                  *      3. non-jailed, non-wild.
1765                  *      4. non-jailed, wild.
1766                  */
1767                 head = &pcbinfo->ipi_wildbase[INP_PCBHASH(INADDR_ANY, lport,
1768                     0, pcbinfo->ipi_wildmask)];
1769                 LIST_FOREACH(inp, head, inp_pcbgroup_wild) {
1770 #ifdef INET6
1771                         /* XXX inp locking */
1772                         if ((inp->inp_vflag & INP_IPV4) == 0)
1773                                 continue;
1774 #endif
1775                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1776                             inp->inp_lport != lport)
1777                                 continue;
1778
1779                         injail = prison_flag(inp->inp_cred, PR_IP4);
1780                         if (injail) {
1781                                 if (prison_check_ip4(inp->inp_cred,
1782                                     &laddr) != 0)
1783                                         continue;
1784                         } else {
1785                                 if (local_exact != NULL)
1786                                         continue;
1787                         }
1788
1789                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1790                                 if (injail)
1791                                         goto found;
1792                                 else
1793                                         local_exact = inp;
1794                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1795 #ifdef INET6
1796                                 /* XXX inp locking, NULL check */
1797                                 if (inp->inp_vflag & INP_IPV6PROTO)
1798                                         local_wild_mapped = inp;
1799                                 else
1800 #endif
1801                                         if (injail)
1802                                                 jail_wild = inp;
1803                                         else
1804                                                 local_wild = inp;
1805                         }
1806                 } /* LIST_FOREACH */
1807                 inp = jail_wild;
1808                 if (inp == NULL)
1809                         inp = local_exact;
1810                 if (inp == NULL)
1811                         inp = local_wild;
1812 #ifdef INET6
1813                 if (inp == NULL)
1814                         inp = local_wild_mapped;
1815 #endif
1816                 if (inp != NULL)
1817                         goto found;
1818         } /* if (lookupflags & INPLOOKUP_WILDCARD) */
1819         INP_GROUP_UNLOCK(pcbgroup);
1820         return (NULL);
1821
1822 found:
1823         in_pcbref(inp);
1824         INP_GROUP_UNLOCK(pcbgroup);
1825         if (lookupflags & INPLOOKUP_WLOCKPCB) {
1826                 INP_WLOCK(inp);
1827                 if (in_pcbrele_wlocked(inp))
1828                         return (NULL);
1829         } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
1830                 INP_RLOCK(inp);
1831                 if (in_pcbrele_rlocked(inp))
1832                         return (NULL);
1833         } else
1834                 panic("%s: locking bug", __func__);
1835         return (inp);
1836 }
1837 #endif /* PCBGROUP */
1838
1839 /*
1840  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
1841  * that the caller has locked the hash list, and will not perform any further
1842  * locking or reference operations on either the hash list or the connection.
1843  */
1844 static struct inpcb *
1845 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1846     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
1847     struct ifnet *ifp)
1848 {
1849         struct inpcbhead *head;
1850         struct inpcb *inp, *tmpinp;
1851         u_short fport = fport_arg, lport = lport_arg;
1852
1853         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1854             ("%s: invalid lookup flags %d", __func__, lookupflags));
1855
1856         INP_HASH_LOCK_ASSERT(pcbinfo);
1857
1858         /*
1859          * First look for an exact match.
1860          */
1861         tmpinp = NULL;
1862         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
1863             pcbinfo->ipi_hashmask)];
1864         LIST_FOREACH(inp, head, inp_hash) {
1865 #ifdef INET6
1866                 /* XXX inp locking */
1867                 if ((inp->inp_vflag & INP_IPV4) == 0)
1868                         continue;
1869 #endif
1870                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
1871                     inp->inp_laddr.s_addr == laddr.s_addr &&
1872                     inp->inp_fport == fport &&
1873                     inp->inp_lport == lport) {
1874                         /*
1875                          * XXX We should be able to directly return
1876                          * the inp here, without any checks.
1877                          * Well unless both bound with SO_REUSEPORT?
1878                          */
1879                         if (prison_flag(inp->inp_cred, PR_IP4))
1880                                 return (inp);
1881                         if (tmpinp == NULL)
1882                                 tmpinp = inp;
1883                 }
1884         }
1885         if (tmpinp != NULL)
1886                 return (tmpinp);
1887
1888         /*
1889          * Then look for a wildcard match, if requested.
1890          */
1891         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1892                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1893 #ifdef INET6
1894                 struct inpcb *local_wild_mapped = NULL;
1895 #endif
1896                 struct inpcb *jail_wild = NULL;
1897                 int injail;
1898
1899                 /*
1900                  * Order of socket selection - we always prefer jails.
1901                  *      1. jailed, non-wild.
1902                  *      2. jailed, wild.
1903                  *      3. non-jailed, non-wild.
1904                  *      4. non-jailed, wild.
1905                  */
1906
1907                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1908                     0, pcbinfo->ipi_hashmask)];
1909                 LIST_FOREACH(inp, head, inp_hash) {
1910 #ifdef INET6
1911                         /* XXX inp locking */
1912                         if ((inp->inp_vflag & INP_IPV4) == 0)
1913                                 continue;
1914 #endif
1915                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1916                             inp->inp_lport != lport)
1917                                 continue;
1918
1919                         injail = prison_flag(inp->inp_cred, PR_IP4);
1920                         if (injail) {
1921                                 if (prison_check_ip4(inp->inp_cred,
1922                                     &laddr) != 0)
1923                                         continue;
1924                         } else {
1925                                 if (local_exact != NULL)
1926                                         continue;
1927                         }
1928
1929                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1930                                 if (injail)
1931                                         return (inp);
1932                                 else
1933                                         local_exact = inp;
1934                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1935 #ifdef INET6
1936                                 /* XXX inp locking, NULL check */
1937                                 if (inp->inp_vflag & INP_IPV6PROTO)
1938                                         local_wild_mapped = inp;
1939                                 else
1940 #endif
1941                                         if (injail)
1942                                                 jail_wild = inp;
1943                                         else
1944                                                 local_wild = inp;
1945                         }
1946                 } /* LIST_FOREACH */
1947                 if (jail_wild != NULL)
1948                         return (jail_wild);
1949                 if (local_exact != NULL)
1950                         return (local_exact);
1951                 if (local_wild != NULL)
1952                         return (local_wild);
1953 #ifdef INET6
1954                 if (local_wild_mapped != NULL)
1955                         return (local_wild_mapped);
1956 #endif
1957         } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
1958
1959         return (NULL);
1960 }
1961
1962 /*
1963  * Lookup PCB in hash list, using pcbinfo tables.  This variation locks the
1964  * hash list lock, and will return the inpcb locked (i.e., requires
1965  * INPLOOKUP_LOCKPCB).
1966  */
1967 static struct inpcb *
1968 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1969     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
1970     struct ifnet *ifp)
1971 {
1972         struct inpcb *inp;
1973
1974         INP_HASH_RLOCK(pcbinfo);
1975         inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
1976             (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
1977         if (inp != NULL) {
1978                 in_pcbref(inp);
1979                 INP_HASH_RUNLOCK(pcbinfo);
1980                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
1981                         INP_WLOCK(inp);
1982                         if (in_pcbrele_wlocked(inp))
1983                                 return (NULL);
1984                 } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
1985                         INP_RLOCK(inp);
1986                         if (in_pcbrele_rlocked(inp))
1987                                 return (NULL);
1988                 } else
1989                         panic("%s: locking bug", __func__);
1990         } else
1991                 INP_HASH_RUNLOCK(pcbinfo);
1992         return (inp);
1993 }
1994
1995 /*
1996  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
1997  * from which a pre-calculated hash value may be extracted.
1998  *
1999  * Possibly more of this logic should be in in_pcbgroup.c.
2000  */
2001 struct inpcb *
2002 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2003     struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp)
2004 {
2005 #if defined(PCBGROUP) && !defined(RSS)
2006         struct inpcbgroup *pcbgroup;
2007 #endif
2008
2009         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2010             ("%s: invalid lookup flags %d", __func__, lookupflags));
2011         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2012             ("%s: LOCKPCB not set", __func__));
2013
2014         /*
2015          * When not using RSS, use connection groups in preference to the
2016          * reservation table when looking up 4-tuples.  When using RSS, just
2017          * use the reservation table, due to the cost of the Toeplitz hash
2018          * in software.
2019          *
2020          * XXXRW: This policy belongs in the pcbgroup code, as in principle
2021          * we could be doing RSS with a non-Toeplitz hash that is affordable
2022          * in software.
2023          */
2024 #if defined(PCBGROUP) && !defined(RSS)
2025         if (in_pcbgroup_enabled(pcbinfo)) {
2026                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2027                     fport);
2028                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2029                     laddr, lport, lookupflags, ifp));
2030         }
2031 #endif
2032         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2033             lookupflags, ifp));
2034 }
2035
2036 struct inpcb *
2037 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2038     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2039     struct ifnet *ifp, struct mbuf *m)
2040 {
2041 #ifdef PCBGROUP
2042         struct inpcbgroup *pcbgroup;
2043 #endif
2044
2045         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2046             ("%s: invalid lookup flags %d", __func__, lookupflags));
2047         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2048             ("%s: LOCKPCB not set", __func__));
2049
2050 #ifdef PCBGROUP
2051         /*
2052          * If we can use a hardware-generated hash to look up the connection
2053          * group, use that connection group to find the inpcb.  Otherwise
2054          * fall back on a software hash -- or the reservation table if we're
2055          * using RSS.
2056          *
2057          * XXXRW: As above, that policy belongs in the pcbgroup code.
2058          */
2059         if (in_pcbgroup_enabled(pcbinfo) &&
2060             !(M_HASHTYPE_TEST(m, M_HASHTYPE_NONE))) {
2061                 pcbgroup = in_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
2062                     m->m_pkthdr.flowid);
2063                 if (pcbgroup != NULL)
2064                         return (in_pcblookup_group(pcbinfo, pcbgroup, faddr,
2065                             fport, laddr, lport, lookupflags, ifp));
2066 #ifndef RSS
2067                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2068                     fport);
2069                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2070                     laddr, lport, lookupflags, ifp));
2071 #endif
2072         }
2073 #endif
2074         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2075             lookupflags, ifp));
2076 }
2077 #endif /* INET */
2078
2079 /*
2080  * Insert PCB onto various hash lists.
2081  */
2082 static int
2083 in_pcbinshash_internal(struct inpcb *inp, int do_pcbgroup_update)
2084 {
2085         struct inpcbhead *pcbhash;
2086         struct inpcbporthead *pcbporthash;
2087         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2088         struct inpcbport *phd;
2089         u_int32_t hashkey_faddr;
2090
2091         INP_WLOCK_ASSERT(inp);
2092         INP_HASH_WLOCK_ASSERT(pcbinfo);
2093
2094         KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2095             ("in_pcbinshash: INP_INHASHLIST"));
2096
2097 #ifdef INET6
2098         if (inp->inp_vflag & INP_IPV6)
2099                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2100         else
2101 #endif
2102         hashkey_faddr = inp->inp_faddr.s_addr;
2103
2104         pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2105                  inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2106
2107         pcbporthash = &pcbinfo->ipi_porthashbase[
2108             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2109
2110         /*
2111          * Go through port list and look for a head for this lport.
2112          */
2113         LIST_FOREACH(phd, pcbporthash, phd_hash) {
2114                 if (phd->phd_port == inp->inp_lport)
2115                         break;
2116         }
2117         /*
2118          * If none exists, malloc one and tack it on.
2119          */
2120         if (phd == NULL) {
2121                 phd = malloc(sizeof(struct inpcbport), M_PCB, M_NOWAIT);
2122                 if (phd == NULL) {
2123                         return (ENOBUFS); /* XXX */
2124                 }
2125                 phd->phd_port = inp->inp_lport;
2126                 LIST_INIT(&phd->phd_pcblist);
2127                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2128         }
2129         inp->inp_phd = phd;
2130         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2131         LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
2132         inp->inp_flags |= INP_INHASHLIST;
2133 #ifdef PCBGROUP
2134         if (do_pcbgroup_update)
2135                 in_pcbgroup_update(inp);
2136 #endif
2137         return (0);
2138 }
2139
2140 /*
2141  * For now, there are two public interfaces to insert an inpcb into the hash
2142  * lists -- one that does update pcbgroups, and one that doesn't.  The latter
2143  * is used only in the TCP syncache, where in_pcbinshash is called before the
2144  * full 4-tuple is set for the inpcb, and we don't want to install in the
2145  * pcbgroup until later.
2146  *
2147  * XXXRW: This seems like a misfeature.  in_pcbinshash should always update
2148  * connection groups, and partially initialised inpcbs should not be exposed
2149  * to either reservation hash tables or pcbgroups.
2150  */
2151 int
2152 in_pcbinshash(struct inpcb *inp)
2153 {
2154
2155         return (in_pcbinshash_internal(inp, 1));
2156 }
2157
2158 int
2159 in_pcbinshash_nopcbgroup(struct inpcb *inp)
2160 {
2161
2162         return (in_pcbinshash_internal(inp, 0));
2163 }
2164
2165 /*
2166  * Move PCB to the proper hash bucket when { faddr, fport } have  been
2167  * changed. NOTE: This does not handle the case of the lport changing (the
2168  * hashed port list would have to be updated as well), so the lport must
2169  * not change after in_pcbinshash() has been called.
2170  */
2171 void
2172 in_pcbrehash_mbuf(struct inpcb *inp, struct mbuf *m)
2173 {
2174         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2175         struct inpcbhead *head;
2176         u_int32_t hashkey_faddr;
2177
2178         INP_WLOCK_ASSERT(inp);
2179         INP_HASH_WLOCK_ASSERT(pcbinfo);
2180
2181         KASSERT(inp->inp_flags & INP_INHASHLIST,
2182             ("in_pcbrehash: !INP_INHASHLIST"));
2183
2184 #ifdef INET6
2185         if (inp->inp_vflag & INP_IPV6)
2186                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2187         else
2188 #endif
2189         hashkey_faddr = inp->inp_faddr.s_addr;
2190
2191         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2192                 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2193
2194         LIST_REMOVE(inp, inp_hash);
2195         LIST_INSERT_HEAD(head, inp, inp_hash);
2196
2197 #ifdef PCBGROUP
2198         if (m != NULL)
2199                 in_pcbgroup_update_mbuf(inp, m);
2200         else
2201                 in_pcbgroup_update(inp);
2202 #endif
2203 }
2204
2205 void
2206 in_pcbrehash(struct inpcb *inp)
2207 {
2208
2209         in_pcbrehash_mbuf(inp, NULL);
2210 }
2211
2212 /*
2213  * Remove PCB from various lists.
2214  */
2215 static void
2216 in_pcbremlists(struct inpcb *inp)
2217 {
2218         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2219
2220 #ifdef INVARIANTS
2221         if (pcbinfo == &V_tcbinfo) {
2222                 INP_INFO_RLOCK_ASSERT(pcbinfo);
2223         } else {
2224                 INP_INFO_WLOCK_ASSERT(pcbinfo);
2225         }
2226 #endif
2227
2228         INP_WLOCK_ASSERT(inp);
2229         INP_LIST_WLOCK_ASSERT(pcbinfo);
2230
2231         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
2232         if (inp->inp_flags & INP_INHASHLIST) {
2233                 struct inpcbport *phd = inp->inp_phd;
2234
2235                 INP_HASH_WLOCK(pcbinfo);
2236                 LIST_REMOVE(inp, inp_hash);
2237                 LIST_REMOVE(inp, inp_portlist);
2238                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
2239                         LIST_REMOVE(phd, phd_hash);
2240                         free(phd, M_PCB);
2241                 }
2242                 INP_HASH_WUNLOCK(pcbinfo);
2243                 inp->inp_flags &= ~INP_INHASHLIST;
2244         }
2245         LIST_REMOVE(inp, inp_list);
2246         pcbinfo->ipi_count--;
2247 #ifdef PCBGROUP
2248         in_pcbgroup_remove(inp);
2249 #endif
2250 }
2251
2252 /*
2253  * Check for alternatives when higher level complains
2254  * about service problems.  For now, invalidate cached
2255  * routing information.  If the route was created dynamically
2256  * (by a redirect), time to try a default gateway again.
2257  */
2258 void
2259 in_losing(struct inpcb *inp)
2260 {
2261
2262         RO_RTFREE(&inp->inp_route);
2263         if (inp->inp_route.ro_lle)
2264                 LLE_FREE(inp->inp_route.ro_lle);        /* zeros ro_lle */
2265         return;
2266 }
2267
2268 /*
2269  * A set label operation has occurred at the socket layer, propagate the
2270  * label change into the in_pcb for the socket.
2271  */
2272 void
2273 in_pcbsosetlabel(struct socket *so)
2274 {
2275 #ifdef MAC
2276         struct inpcb *inp;
2277
2278         inp = sotoinpcb(so);
2279         KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2280
2281         INP_WLOCK(inp);
2282         SOCK_LOCK(so);
2283         mac_inpcb_sosetlabel(so, inp);
2284         SOCK_UNLOCK(so);
2285         INP_WUNLOCK(inp);
2286 #endif
2287 }
2288
2289 /*
2290  * ipport_tick runs once per second, determining if random port allocation
2291  * should be continued.  If more than ipport_randomcps ports have been
2292  * allocated in the last second, then we return to sequential port
2293  * allocation. We return to random allocation only once we drop below
2294  * ipport_randomcps for at least ipport_randomtime seconds.
2295  */
2296 static void
2297 ipport_tick(void *xtp)
2298 {
2299         VNET_ITERATOR_DECL(vnet_iter);
2300
2301         VNET_LIST_RLOCK_NOSLEEP();
2302         VNET_FOREACH(vnet_iter) {
2303                 CURVNET_SET(vnet_iter); /* XXX appease INVARIANTS here */
2304                 if (V_ipport_tcpallocs <=
2305                     V_ipport_tcplastcount + V_ipport_randomcps) {
2306                         if (V_ipport_stoprandom > 0)
2307                                 V_ipport_stoprandom--;
2308                 } else
2309                         V_ipport_stoprandom = V_ipport_randomtime;
2310                 V_ipport_tcplastcount = V_ipport_tcpallocs;
2311                 CURVNET_RESTORE();
2312         }
2313         VNET_LIST_RUNLOCK_NOSLEEP();
2314         callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
2315 }
2316
2317 static void
2318 ip_fini(void *xtp)
2319 {
2320
2321         callout_stop(&ipport_tick_callout);
2322 }
2323
2324 /*
2325  * The ipport_callout should start running at about the time we attach the
2326  * inet or inet6 domains.
2327  */
2328 static void
2329 ipport_tick_init(const void *unused __unused)
2330 {
2331
2332         /* Start ipport_tick. */
2333         callout_init(&ipport_tick_callout, 1);
2334         callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
2335         EVENTHANDLER_REGISTER(shutdown_pre_sync, ip_fini, NULL,
2336                 SHUTDOWN_PRI_DEFAULT);
2337 }
2338 SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
2339     ipport_tick_init, NULL);
2340
2341 void
2342 inp_wlock(struct inpcb *inp)
2343 {
2344
2345         INP_WLOCK(inp);
2346 }
2347
2348 void
2349 inp_wunlock(struct inpcb *inp)
2350 {
2351
2352         INP_WUNLOCK(inp);
2353 }
2354
2355 void
2356 inp_rlock(struct inpcb *inp)
2357 {
2358
2359         INP_RLOCK(inp);
2360 }
2361
2362 void
2363 inp_runlock(struct inpcb *inp)
2364 {
2365
2366         INP_RUNLOCK(inp);
2367 }
2368
2369 #ifdef INVARIANT_SUPPORT
2370 void
2371 inp_lock_assert(struct inpcb *inp)
2372 {
2373
2374         INP_WLOCK_ASSERT(inp);
2375 }
2376
2377 void
2378 inp_unlock_assert(struct inpcb *inp)
2379 {
2380
2381         INP_UNLOCK_ASSERT(inp);
2382 }
2383 #endif
2384
2385 void
2386 inp_apply_all(void (*func)(struct inpcb *, void *), void *arg)
2387 {
2388         struct inpcb *inp;
2389
2390         INP_INFO_WLOCK(&V_tcbinfo);
2391         LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) {
2392                 INP_WLOCK(inp);
2393                 func(inp, arg);
2394                 INP_WUNLOCK(inp);
2395         }
2396         INP_INFO_WUNLOCK(&V_tcbinfo);
2397 }
2398
2399 struct socket *
2400 inp_inpcbtosocket(struct inpcb *inp)
2401 {
2402
2403         INP_WLOCK_ASSERT(inp);
2404         return (inp->inp_socket);
2405 }
2406
2407 struct tcpcb *
2408 inp_inpcbtotcpcb(struct inpcb *inp)
2409 {
2410
2411         INP_WLOCK_ASSERT(inp);
2412         return ((struct tcpcb *)inp->inp_ppcb);
2413 }
2414
2415 int
2416 inp_ip_tos_get(const struct inpcb *inp)
2417 {
2418
2419         return (inp->inp_ip_tos);
2420 }
2421
2422 void
2423 inp_ip_tos_set(struct inpcb *inp, int val)
2424 {
2425
2426         inp->inp_ip_tos = val;
2427 }
2428
2429 void
2430 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2431     uint32_t *faddr, uint16_t *fp)
2432 {
2433
2434         INP_LOCK_ASSERT(inp);
2435         *laddr = inp->inp_laddr.s_addr;
2436         *faddr = inp->inp_faddr.s_addr;
2437         *lp = inp->inp_lport;
2438         *fp = inp->inp_fport;
2439 }
2440
2441 struct inpcb *
2442 so_sotoinpcb(struct socket *so)
2443 {
2444
2445         return (sotoinpcb(so));
2446 }
2447
2448 struct tcpcb *
2449 so_sototcpcb(struct socket *so)
2450 {
2451
2452         return (sototcpcb(so));
2453 }
2454
2455 /*
2456  * Create an external-format (``xinpcb'') structure using the information in
2457  * the kernel-format in_pcb structure pointed to by inp.  This is done to
2458  * reduce the spew of irrelevant information over this interface, to isolate
2459  * user code from changes in the kernel structure, and potentially to provide
2460  * information-hiding if we decide that some of this information should be
2461  * hidden from users.
2462  */
2463 void
2464 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2465 {
2466
2467         xi->xi_len = sizeof(struct xinpcb);
2468         if (inp->inp_socket)
2469                 sotoxsocket(inp->inp_socket, &xi->xi_socket);
2470         else
2471                 bzero(&xi->xi_socket, sizeof(struct xsocket));
2472         bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2473         xi->inp_gencnt = inp->inp_gencnt;
2474         xi->inp_ppcb = inp->inp_ppcb;
2475         xi->inp_flow = inp->inp_flow;
2476         xi->inp_flowid = inp->inp_flowid;
2477         xi->inp_flowtype = inp->inp_flowtype;
2478         xi->inp_flags = inp->inp_flags;
2479         xi->inp_flags2 = inp->inp_flags2;
2480         xi->inp_rss_listen_bucket = inp->inp_rss_listen_bucket;
2481         xi->in6p_cksum = inp->in6p_cksum;
2482         xi->in6p_hops = inp->in6p_hops;
2483         xi->inp_ip_tos = inp->inp_ip_tos;
2484         xi->inp_vflag = inp->inp_vflag;
2485         xi->inp_ip_ttl = inp->inp_ip_ttl;
2486         xi->inp_ip_p = inp->inp_ip_p;
2487         xi->inp_ip_minttl = inp->inp_ip_minttl;
2488 }
2489
2490 #ifdef DDB
2491 static void
2492 db_print_indent(int indent)
2493 {
2494         int i;
2495
2496         for (i = 0; i < indent; i++)
2497                 db_printf(" ");
2498 }
2499
2500 static void
2501 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
2502 {
2503         char faddr_str[48], laddr_str[48];
2504
2505         db_print_indent(indent);
2506         db_printf("%s at %p\n", name, inc);
2507
2508         indent += 2;
2509
2510 #ifdef INET6
2511         if (inc->inc_flags & INC_ISIPV6) {
2512                 /* IPv6. */
2513                 ip6_sprintf(laddr_str, &inc->inc6_laddr);
2514                 ip6_sprintf(faddr_str, &inc->inc6_faddr);
2515         } else
2516 #endif
2517         {
2518                 /* IPv4. */
2519                 inet_ntoa_r(inc->inc_laddr, laddr_str);
2520                 inet_ntoa_r(inc->inc_faddr, faddr_str);
2521         }
2522         db_print_indent(indent);
2523         db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
2524             ntohs(inc->inc_lport));
2525         db_print_indent(indent);
2526         db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
2527             ntohs(inc->inc_fport));
2528 }
2529
2530 static void
2531 db_print_inpflags(int inp_flags)
2532 {
2533         int comma;
2534
2535         comma = 0;
2536         if (inp_flags & INP_RECVOPTS) {
2537                 db_printf("%sINP_RECVOPTS", comma ? ", " : "");
2538                 comma = 1;
2539         }
2540         if (inp_flags & INP_RECVRETOPTS) {
2541                 db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
2542                 comma = 1;
2543         }
2544         if (inp_flags & INP_RECVDSTADDR) {
2545                 db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
2546                 comma = 1;
2547         }
2548         if (inp_flags & INP_ORIGDSTADDR) {
2549                 db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
2550                 comma = 1;
2551         }
2552         if (inp_flags & INP_HDRINCL) {
2553                 db_printf("%sINP_HDRINCL", comma ? ", " : "");
2554                 comma = 1;
2555         }
2556         if (inp_flags & INP_HIGHPORT) {
2557                 db_printf("%sINP_HIGHPORT", comma ? ", " : "");
2558                 comma = 1;
2559         }
2560         if (inp_flags & INP_LOWPORT) {
2561                 db_printf("%sINP_LOWPORT", comma ? ", " : "");
2562                 comma = 1;
2563         }
2564         if (inp_flags & INP_ANONPORT) {
2565                 db_printf("%sINP_ANONPORT", comma ? ", " : "");
2566                 comma = 1;
2567         }
2568         if (inp_flags & INP_RECVIF) {
2569                 db_printf("%sINP_RECVIF", comma ? ", " : "");
2570                 comma = 1;
2571         }
2572         if (inp_flags & INP_MTUDISC) {
2573                 db_printf("%sINP_MTUDISC", comma ? ", " : "");
2574                 comma = 1;
2575         }
2576         if (inp_flags & INP_RECVTTL) {
2577                 db_printf("%sINP_RECVTTL", comma ? ", " : "");
2578                 comma = 1;
2579         }
2580         if (inp_flags & INP_DONTFRAG) {
2581                 db_printf("%sINP_DONTFRAG", comma ? ", " : "");
2582                 comma = 1;
2583         }
2584         if (inp_flags & INP_RECVTOS) {
2585                 db_printf("%sINP_RECVTOS", comma ? ", " : "");
2586                 comma = 1;
2587         }
2588         if (inp_flags & IN6P_IPV6_V6ONLY) {
2589                 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
2590                 comma = 1;
2591         }
2592         if (inp_flags & IN6P_PKTINFO) {
2593                 db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
2594                 comma = 1;
2595         }
2596         if (inp_flags & IN6P_HOPLIMIT) {
2597                 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
2598                 comma = 1;
2599         }
2600         if (inp_flags & IN6P_HOPOPTS) {
2601                 db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
2602                 comma = 1;
2603         }
2604         if (inp_flags & IN6P_DSTOPTS) {
2605                 db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
2606                 comma = 1;
2607         }
2608         if (inp_flags & IN6P_RTHDR) {
2609                 db_printf("%sIN6P_RTHDR", comma ? ", " : "");
2610                 comma = 1;
2611         }
2612         if (inp_flags & IN6P_RTHDRDSTOPTS) {
2613                 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
2614                 comma = 1;
2615         }
2616         if (inp_flags & IN6P_TCLASS) {
2617                 db_printf("%sIN6P_TCLASS", comma ? ", " : "");
2618                 comma = 1;
2619         }
2620         if (inp_flags & IN6P_AUTOFLOWLABEL) {
2621                 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
2622                 comma = 1;
2623         }
2624         if (inp_flags & INP_TIMEWAIT) {
2625                 db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
2626                 comma  = 1;
2627         }
2628         if (inp_flags & INP_ONESBCAST) {
2629                 db_printf("%sINP_ONESBCAST", comma ? ", " : "");
2630                 comma  = 1;
2631         }
2632         if (inp_flags & INP_DROPPED) {
2633                 db_printf("%sINP_DROPPED", comma ? ", " : "");
2634                 comma  = 1;
2635         }
2636         if (inp_flags & INP_SOCKREF) {
2637                 db_printf("%sINP_SOCKREF", comma ? ", " : "");
2638                 comma  = 1;
2639         }
2640         if (inp_flags & IN6P_RFC2292) {
2641                 db_printf("%sIN6P_RFC2292", comma ? ", " : "");
2642                 comma = 1;
2643         }
2644         if (inp_flags & IN6P_MTU) {
2645                 db_printf("IN6P_MTU%s", comma ? ", " : "");
2646                 comma = 1;
2647         }
2648 }
2649
2650 static void
2651 db_print_inpvflag(u_char inp_vflag)
2652 {
2653         int comma;
2654
2655         comma = 0;
2656         if (inp_vflag & INP_IPV4) {
2657                 db_printf("%sINP_IPV4", comma ? ", " : "");
2658                 comma  = 1;
2659         }
2660         if (inp_vflag & INP_IPV6) {
2661                 db_printf("%sINP_IPV6", comma ? ", " : "");
2662                 comma  = 1;
2663         }
2664         if (inp_vflag & INP_IPV6PROTO) {
2665                 db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
2666                 comma  = 1;
2667         }
2668 }
2669
2670 static void
2671 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
2672 {
2673
2674         db_print_indent(indent);
2675         db_printf("%s at %p\n", name, inp);
2676
2677         indent += 2;
2678
2679         db_print_indent(indent);
2680         db_printf("inp_flow: 0x%x\n", inp->inp_flow);
2681
2682         db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
2683
2684         db_print_indent(indent);
2685         db_printf("inp_ppcb: %p   inp_pcbinfo: %p   inp_socket: %p\n",
2686             inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
2687
2688         db_print_indent(indent);
2689         db_printf("inp_label: %p   inp_flags: 0x%x (",
2690            inp->inp_label, inp->inp_flags);
2691         db_print_inpflags(inp->inp_flags);
2692         db_printf(")\n");
2693
2694         db_print_indent(indent);
2695         db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
2696             inp->inp_vflag);
2697         db_print_inpvflag(inp->inp_vflag);
2698         db_printf(")\n");
2699
2700         db_print_indent(indent);
2701         db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
2702             inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
2703
2704         db_print_indent(indent);
2705 #ifdef INET6
2706         if (inp->inp_vflag & INP_IPV6) {
2707                 db_printf("in6p_options: %p   in6p_outputopts: %p   "
2708                     "in6p_moptions: %p\n", inp->in6p_options,
2709                     inp->in6p_outputopts, inp->in6p_moptions);
2710                 db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
2711                     "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
2712                     inp->in6p_hops);
2713         } else
2714 #endif
2715         {
2716                 db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
2717                     "inp_ip_moptions: %p\n", inp->inp_ip_tos,
2718                     inp->inp_options, inp->inp_moptions);
2719         }
2720
2721         db_print_indent(indent);
2722         db_printf("inp_phd: %p   inp_gencnt: %ju\n", inp->inp_phd,
2723             (uintmax_t)inp->inp_gencnt);
2724 }
2725
2726 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
2727 {
2728         struct inpcb *inp;
2729
2730         if (!have_addr) {
2731                 db_printf("usage: show inpcb <addr>\n");
2732                 return;
2733         }
2734         inp = (struct inpcb *)addr;
2735
2736         db_print_inpcb(inp, "inpcb", 0);
2737 }
2738 #endif /* DDB */
2739
2740 #ifdef RATELIMIT
2741 /*
2742  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
2743  * if any.
2744  */
2745 int
2746 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
2747 {
2748         union if_snd_tag_modify_params params = {
2749                 .rate_limit.max_rate = max_pacing_rate,
2750         };
2751         struct m_snd_tag *mst;
2752         struct ifnet *ifp;
2753         int error;
2754
2755         mst = inp->inp_snd_tag;
2756         if (mst == NULL)
2757                 return (EINVAL);
2758
2759         ifp = mst->ifp;
2760         if (ifp == NULL)
2761                 return (EINVAL);
2762
2763         if (ifp->if_snd_tag_modify == NULL) {
2764                 error = EOPNOTSUPP;
2765         } else {
2766                 error = ifp->if_snd_tag_modify(mst, &params);
2767         }
2768         return (error);
2769 }
2770
2771 /*
2772  * Query existing TX rate limit based on the existing
2773  * "inp->inp_snd_tag", if any.
2774  */
2775 int
2776 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
2777 {
2778         union if_snd_tag_query_params params = { };
2779         struct m_snd_tag *mst;
2780         struct ifnet *ifp;
2781         int error;
2782
2783         mst = inp->inp_snd_tag;
2784         if (mst == NULL)
2785                 return (EINVAL);
2786
2787         ifp = mst->ifp;
2788         if (ifp == NULL)
2789                 return (EINVAL);
2790
2791         if (ifp->if_snd_tag_query == NULL) {
2792                 error = EOPNOTSUPP;
2793         } else {
2794                 error = ifp->if_snd_tag_query(mst, &params);
2795                 if (error == 0 &&  p_max_pacing_rate != NULL)
2796                         *p_max_pacing_rate = params.rate_limit.max_rate;
2797         }
2798         return (error);
2799 }
2800
2801 /*
2802  * Query existing TX queue level based on the existing
2803  * "inp->inp_snd_tag", if any.
2804  */
2805 int
2806 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
2807 {
2808         union if_snd_tag_query_params params = { };
2809         struct m_snd_tag *mst;
2810         struct ifnet *ifp;
2811         int error;
2812
2813         mst = inp->inp_snd_tag;
2814         if (mst == NULL)
2815                 return (EINVAL);
2816
2817         ifp = mst->ifp;
2818         if (ifp == NULL)
2819                 return (EINVAL);
2820
2821         if (ifp->if_snd_tag_query == NULL)
2822                 return (EOPNOTSUPP);
2823
2824         error = ifp->if_snd_tag_query(mst, &params);
2825         if (error == 0 &&  p_txqueue_level != NULL)
2826                 *p_txqueue_level = params.rate_limit.queue_level;
2827         return (error);
2828 }
2829
2830 /*
2831  * Allocate a new TX rate limit send tag from the network interface
2832  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
2833  */
2834 int
2835 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
2836     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate)
2837 {
2838         union if_snd_tag_alloc_params params = {
2839                 .rate_limit.hdr.type = (max_pacing_rate == -1U) ?
2840                     IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
2841                 .rate_limit.hdr.flowid = flowid,
2842                 .rate_limit.hdr.flowtype = flowtype,
2843                 .rate_limit.max_rate = max_pacing_rate,
2844         };
2845         int error;
2846
2847         INP_WLOCK_ASSERT(inp);
2848
2849         if (inp->inp_snd_tag != NULL)
2850                 return (EINVAL);
2851
2852         if (ifp->if_snd_tag_alloc == NULL) {
2853                 error = EOPNOTSUPP;
2854         } else {
2855                 error = ifp->if_snd_tag_alloc(ifp, &params, &inp->inp_snd_tag);
2856
2857                 /*
2858                  * At success increment the refcount on
2859                  * the send tag's network interface:
2860                  */
2861                 if (error == 0)
2862                         if_ref(inp->inp_snd_tag->ifp);
2863         }
2864         return (error);
2865 }
2866
2867 /*
2868  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
2869  * if any:
2870  */
2871 void
2872 in_pcbdetach_txrtlmt(struct inpcb *inp)
2873 {
2874         struct m_snd_tag *mst;
2875         struct ifnet *ifp;
2876
2877         INP_WLOCK_ASSERT(inp);
2878
2879         mst = inp->inp_snd_tag;
2880         inp->inp_snd_tag = NULL;
2881
2882         if (mst == NULL)
2883                 return;
2884
2885         ifp = mst->ifp;
2886         if (ifp == NULL)
2887                 return;
2888
2889         /*
2890          * If the device was detached while we still had reference(s)
2891          * on the ifp, we assume if_snd_tag_free() was replaced with
2892          * stubs.
2893          */
2894         ifp->if_snd_tag_free(mst);
2895
2896         /* release reference count on network interface */
2897         if_rele(ifp);
2898 }
2899
2900 /*
2901  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
2902  * is set in the fast path and will attach/detach/modify the TX rate
2903  * limit send tag based on the socket's so_max_pacing_rate value.
2904  */
2905 void
2906 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
2907 {
2908         struct socket *socket;
2909         uint32_t max_pacing_rate;
2910         bool did_upgrade;
2911         int error;
2912
2913         if (inp == NULL)
2914                 return;
2915
2916         socket = inp->inp_socket;
2917         if (socket == NULL)
2918                 return;
2919
2920         if (!INP_WLOCKED(inp)) {
2921                 /*
2922                  * NOTE: If the write locking fails, we need to bail
2923                  * out and use the non-ratelimited ring for the
2924                  * transmit until there is a new chance to get the
2925                  * write lock.
2926                  */
2927                 if (!INP_TRY_UPGRADE(inp))
2928                         return;
2929                 did_upgrade = 1;
2930         } else {
2931                 did_upgrade = 0;
2932         }
2933
2934         /*
2935          * NOTE: The so_max_pacing_rate value is read unlocked,
2936          * because atomic updates are not required since the variable
2937          * is checked at every mbuf we send. It is assumed that the
2938          * variable read itself will be atomic.
2939          */
2940         max_pacing_rate = socket->so_max_pacing_rate;
2941
2942         /*
2943          * NOTE: When attaching to a network interface a reference is
2944          * made to ensure the network interface doesn't go away until
2945          * all ratelimit connections are gone. The network interface
2946          * pointers compared below represent valid network interfaces,
2947          * except when comparing towards NULL.
2948          */
2949         if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
2950                 error = 0;
2951         } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
2952                 if (inp->inp_snd_tag != NULL)
2953                         in_pcbdetach_txrtlmt(inp);
2954                 error = 0;
2955         } else if (inp->inp_snd_tag == NULL) {
2956                 /*
2957                  * In order to utilize packet pacing with RSS, we need
2958                  * to wait until there is a valid RSS hash before we
2959                  * can proceed:
2960                  */
2961                 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
2962                         error = EAGAIN;
2963                 } else {
2964                         error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
2965                             mb->m_pkthdr.flowid, max_pacing_rate);
2966                 }
2967         } else {
2968                 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
2969         }
2970         if (error == 0 || error == EOPNOTSUPP)
2971                 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
2972         if (did_upgrade)
2973                 INP_DOWNGRADE(inp);
2974 }
2975
2976 /*
2977  * Track route changes for TX rate limiting.
2978  */
2979 void
2980 in_pcboutput_eagain(struct inpcb *inp)
2981 {
2982         struct socket *socket;
2983         bool did_upgrade;
2984
2985         if (inp == NULL)
2986                 return;
2987
2988         socket = inp->inp_socket;
2989         if (socket == NULL)
2990                 return;
2991
2992         if (inp->inp_snd_tag == NULL)
2993                 return;
2994
2995         if (!INP_WLOCKED(inp)) {
2996                 /*
2997                  * NOTE: If the write locking fails, we need to bail
2998                  * out and use the non-ratelimited ring for the
2999                  * transmit until there is a new chance to get the
3000                  * write lock.
3001                  */
3002                 if (!INP_TRY_UPGRADE(inp))
3003                         return;
3004                 did_upgrade = 1;
3005         } else {
3006                 did_upgrade = 0;
3007         }
3008
3009         /* detach rate limiting */
3010         in_pcbdetach_txrtlmt(inp);
3011
3012         /* make sure new mbuf send tag allocation is made */
3013         inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3014
3015         if (did_upgrade)
3016                 INP_DOWNGRADE(inp);
3017 }
3018 #endif /* RATELIMIT */