sys/netinet/in_pcb.c

   1 /*-
   2  * Copyright (c) 1982, 1986, 1991, 1993, 1995
   3  *      The Regents of the University of California.
   4  * Copyright (c) 2007-2009 Robert N. M. Watson
   5  * Copyright (c) 2010-2011 Juniper Networks, Inc.
   6  * All rights reserved.
   7  *
   8  * Portions of this software were developed by Robert N. M. Watson under
   9  * contract to Juniper Networks, Inc.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 4. Neither the name of the University nor the names of its contributors
  20  *    may be used to endorse or promote products derived from this software
  21  *    without specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  26  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  27  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  31  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  32  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  *
  35  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  36  */
  37
  38 #include <sys/cdefs.h>
  39 __FBSDID("$FreeBSD$");
  40
  41 #include "opt_ddb.h"
  42 #include "opt_ipsec.h"
  43 #include "opt_inet.h"
  44 #include "opt_inet6.h"
  45 #include "opt_ratelimit.h"
  46 #include "opt_pcbgroup.h"
  47 #include "opt_rss.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/systm.h>
  51 #include <sys/lock.h>
  52 #include <sys/malloc.h>
  53 #include <sys/mbuf.h>
  54 #include <sys/callout.h>
  55 #include <sys/eventhandler.h>
  56 #include <sys/domain.h>
  57 #include <sys/protosw.h>
  58 #include <sys/rmlock.h>
  59 #include <sys/socket.h>
  60 #include <sys/socketvar.h>
  61 #include <sys/sockio.h>
  62 #include <sys/priv.h>
  63 #include <sys/proc.h>
  64 #include <sys/refcount.h>
  65 #include <sys/jail.h>
  66 #include <sys/kernel.h>
  67 #include <sys/sysctl.h>
  68
  69 #ifdef DDB
  70 #include <ddb/ddb.h>
  71 #endif
  72
  73 #include <vm/uma.h>
  74
  75 #include <net/if.h>
  76 #include <net/if_var.h>
  77 #include <net/if_types.h>
  78 #include <net/if_llatbl.h>
  79 #include <net/route.h>
  80 #include <net/rss_config.h>
  81 #include <net/vnet.h>
  82
  83 #if defined(INET) || defined(INET6)
  84 #include <netinet/in.h>
  85 #include <netinet/in_pcb.h>
  86 #include <netinet/ip_var.h>
  87 #include <netinet/tcp_var.h>
  88 #include <netinet/udp.h>
  89 #include <netinet/udp_var.h>
  90 #endif
  91 #ifdef INET
  92 #include <netinet/in_var.h>
  93 #endif
  94 #ifdef INET6
  95 #include <netinet/ip6.h>
  96 #include <netinet6/in6_pcb.h>
  97 #include <netinet6/in6_var.h>
  98 #include <netinet6/ip6_var.h>
  99 #endif /* INET6 */
 100
 101
 102 #ifdef IPSEC
 103 #include <netipsec/ipsec.h>
 104 #include <netipsec/key.h>
 105 #endif /* IPSEC */
 106
 107 #include <security/mac/mac_framework.h>
 108
 109 static struct callout   ipport_tick_callout;
 110
 111 /*
 112  * These configure the range of local port addresses assigned to
 113  * "unspecified" outgoing connections/packets/whatever.
 114  */
 115 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;    /* 1023 */
 116 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;    /* 600 */
 117 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;     /* 10000 */
 118 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;       /* 65535 */
 119 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;      /* 49152 */
 120 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;        /* 65535 */
 121
 122 /*
 123  * Reserved ports accessible only to root. There are significant
 124  * security considerations that must be accounted for when changing these,
 125  * but the security benefits can be great. Please be careful.
 126  */
 127 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;    /* 1023 */
 128 VNET_DEFINE(int, ipport_reservedlow);
 129
 130 /* Variables dealing with random ephemeral port allocation. */
 131 VNET_DEFINE(int, ipport_randomized) = 1;        /* user controlled via sysctl */
 132 VNET_DEFINE(int, ipport_randomcps) = 10;        /* user controlled via sysctl */
 133 VNET_DEFINE(int, ipport_randomtime) = 45;       /* user controlled via sysctl */
 134 VNET_DEFINE(int, ipport_stoprandom);            /* toggled by ipport_tick */
 135 VNET_DEFINE(int, ipport_tcpallocs);
 136 static VNET_DEFINE(int, ipport_tcplastcount);
 137
 138 #define V_ipport_tcplastcount           VNET(ipport_tcplastcount)
 139
 140 static void     in_pcbremlists(struct inpcb *inp);
 141 #ifdef INET
 142 static struct inpcb     *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
 143                             struct in_addr faddr, u_int fport_arg,
 144                             struct in_addr laddr, u_int lport_arg,
 145                             int lookupflags, struct ifnet *ifp);
 146
 147 #define RANGECHK(var, min, max) \
 148         if ((var) < (min)) { (var) = (min); } \
 149         else if ((var) > (max)) { (var) = (max); }
 150
 151 static int
 152 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 153 {
 154         int error;
 155
 156         error = sysctl_handle_int(oidp, arg1, arg2, req);
 157         if (error == 0) {
 158                 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 159                 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 160                 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
 161                 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
 162                 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
 163                 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
 164         }
 165         return (error);
 166 }
 167
 168 #undef RANGECHK
 169
 170 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0,
 171     "IP Ports");
 172
 173 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
 174         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 175         &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I", "");
 176 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
 177         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 178         &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I", "");
 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
 180         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 181         &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I", "");
 182 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
 183         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 184         &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I", "");
 185 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
 186         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 187         &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I", "");
 188 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
 189         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 190         &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I", "");
 191 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
 192         CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
 193         &VNET_NAME(ipport_reservedhigh), 0, "");
 194 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
 195         CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
 196 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
 197         CTLFLAG_VNET | CTLFLAG_RW,
 198         &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
 199 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
 200         CTLFLAG_VNET | CTLFLAG_RW,
 201         &VNET_NAME(ipport_randomcps), 0, "Maximum number of random port "
 202         "allocations before switching to a sequental one");
 203 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
 204         CTLFLAG_VNET | CTLFLAG_RW,
 205         &VNET_NAME(ipport_randomtime), 0,
 206         "Minimum time to keep sequental port "
 207         "allocation before switching to a random one");
 208 #endif /* INET */
 209
 210 /*
 211  * in_pcb.c: manage the Protocol Control Blocks.
 212  *
 213  * NOTE: It is assumed that most of these functions will be called with
 214  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
 215  * functions often modify hash chains or addresses in pcbs.
 216  */
 217
 218 /*
 219  * Initialize an inpcbinfo -- we should be able to reduce the number of
 220  * arguments in time.
 221  */
 222 void
 223 in_pcbinfo_init(struct inpcbinfo *pcbinfo, const char *name,
 224     struct inpcbhead *listhead, int hash_nelements, int porthash_nelements,
 225     char *inpcbzone_name, uma_init inpcbzone_init, uma_fini inpcbzone_fini,
 226     uint32_t inpcbzone_flags, u_int hashfields)
 227 {
 228
 229         INP_INFO_LOCK_INIT(pcbinfo, name);
 230         INP_HASH_LOCK_INIT(pcbinfo, "pcbinfohash");     /* XXXRW: argument? */
 231         INP_LIST_LOCK_INIT(pcbinfo, "pcbinfolist");
 232 #ifdef VIMAGE
 233         pcbinfo->ipi_vnet = curvnet;
 234 #endif
 235         pcbinfo->ipi_listhead = listhead;
 236         LIST_INIT(pcbinfo->ipi_listhead);
 237         pcbinfo->ipi_count = 0;
 238         pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB,
 239             &pcbinfo->ipi_hashmask);
 240         pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
 241             &pcbinfo->ipi_porthashmask);
 242 #ifdef PCBGROUP
 243         in_pcbgroup_init(pcbinfo, hashfields, hash_nelements);
 244 #endif
 245         pcbinfo->ipi_zone = uma_zcreate(inpcbzone_name, sizeof(struct inpcb),
 246             NULL, NULL, inpcbzone_init, inpcbzone_fini, UMA_ALIGN_PTR,
 247             inpcbzone_flags);
 248         uma_zone_set_max(pcbinfo->ipi_zone, maxsockets);
 249         uma_zone_set_warning(pcbinfo->ipi_zone,
 250             "kern.ipc.maxsockets limit reached");
 251 }
 252
 253 /*
 254  * Destroy an inpcbinfo.
 255  */
 256 void
 257 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
 258 {
 259
 260         KASSERT(pcbinfo->ipi_count == 0,
 261             ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
 262
 263         hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask);
 264         hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
 265             pcbinfo->ipi_porthashmask);
 266 #ifdef PCBGROUP
 267         in_pcbgroup_destroy(pcbinfo);
 268 #endif
 269         uma_zdestroy(pcbinfo->ipi_zone);
 270         INP_LIST_LOCK_DESTROY(pcbinfo);
 271         INP_HASH_LOCK_DESTROY(pcbinfo);
 272         INP_INFO_LOCK_DESTROY(pcbinfo);
 273 }
 274
 275 /*
 276  * Allocate a PCB and associate it with the socket.
 277  * On success return with the PCB locked.
 278  */
 279 int
 280 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 281 {
 282         struct inpcb *inp;
 283         int error;
 284
 285 #ifdef INVARIANTS
 286         if (pcbinfo == &V_tcbinfo) {
 287                 INP_INFO_RLOCK_ASSERT(pcbinfo);
 288         } else {
 289                 INP_INFO_WLOCK_ASSERT(pcbinfo);
 290         }
 291 #endif
 292
 293         error = 0;
 294         inp = uma_zalloc(pcbinfo->ipi_zone, M_NOWAIT);
 295         if (inp == NULL)
 296                 return (ENOBUFS);
 297         bzero(inp, inp_zero_size);
 298         inp->inp_pcbinfo = pcbinfo;
 299         inp->inp_socket = so;
 300         inp->inp_cred = crhold(so->so_cred);
 301         inp->inp_inc.inc_fibnum = so->so_fibnum;
 302 #ifdef MAC
 303         error = mac_inpcb_init(inp, M_NOWAIT);
 304         if (error != 0)
 305                 goto out;
 306         mac_inpcb_create(so, inp);
 307 #endif
 308 #ifdef IPSEC
 309         error = ipsec_init_policy(so, &inp->inp_sp);
 310         if (error != 0) {
 311 #ifdef MAC
 312                 mac_inpcb_destroy(inp);
 313 #endif
 314                 goto out;
 315         }
 316 #endif /*IPSEC*/
 317 #ifdef INET6
 318         if (INP_SOCKAF(so) == AF_INET6) {
 319                 inp->inp_vflag |= INP_IPV6PROTO;
 320                 if (V_ip6_v6only)
 321                         inp->inp_flags |= IN6P_IPV6_V6ONLY;
 322         }
 323 #endif
 324         INP_WLOCK(inp);
 325         INP_LIST_WLOCK(pcbinfo);
 326         LIST_INSERT_HEAD(pcbinfo->ipi_listhead, inp, inp_list);
 327         pcbinfo->ipi_count++;
 328         so->so_pcb = (caddr_t)inp;
 329 #ifdef INET6
 330         if (V_ip6_auto_flowlabel)
 331                 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 332 #endif
 333         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 334         refcount_init(&inp->inp_refcount, 1);   /* Reference from inpcbinfo */
 335         INP_LIST_WUNLOCK(pcbinfo);
 336 #if defined(IPSEC) || defined(MAC)
 337 out:
 338         if (error != 0) {
 339                 crfree(inp->inp_cred);
 340                 uma_zfree(pcbinfo->ipi_zone, inp);
 341         }
 342 #endif
 343         return (error);
 344 }
 345
 346 #ifdef INET
 347 int
 348 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 349 {
 350         int anonport, error;
 351
 352         INP_WLOCK_ASSERT(inp);
 353         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 354
 355         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 356                 return (EINVAL);
 357         anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
 358         error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
 359             &inp->inp_lport, cred);
 360         if (error)
 361                 return (error);
 362         if (in_pcbinshash(inp) != 0) {
 363                 inp->inp_laddr.s_addr = INADDR_ANY;
 364                 inp->inp_lport = 0;
 365                 return (EAGAIN);
 366         }
 367         if (anonport)
 368                 inp->inp_flags |= INP_ANONPORT;
 369         return (0);
 370 }
 371 #endif
 372
 373 /*
 374  * Select a local port (number) to use.
 375  */
 376 #if defined(INET) || defined(INET6)
 377 int
 378 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
 379     struct ucred *cred, int lookupflags)
 380 {
 381         struct inpcbinfo *pcbinfo;
 382         struct inpcb *tmpinp;
 383         unsigned short *lastport;
 384         int count, dorandom, error;
 385         u_short aux, first, last, lport;
 386 #ifdef INET
 387         struct in_addr laddr;
 388 #endif
 389
 390         pcbinfo = inp->inp_pcbinfo;
 391
 392         /*
 393          * Because no actual state changes occur here, a global write lock on
 394          * the pcbinfo isn't required.
 395          */
 396         INP_LOCK_ASSERT(inp);
 397         INP_HASH_LOCK_ASSERT(pcbinfo);
 398
 399         if (inp->inp_flags & INP_HIGHPORT) {
 400                 first = V_ipport_hifirstauto;   /* sysctl */
 401                 last  = V_ipport_hilastauto;
 402                 lastport = &pcbinfo->ipi_lasthi;
 403         } else if (inp->inp_flags & INP_LOWPORT) {
 404                 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0);
 405                 if (error)
 406                         return (error);
 407                 first = V_ipport_lowfirstauto;  /* 1023 */
 408                 last  = V_ipport_lowlastauto;   /* 600 */
 409                 lastport = &pcbinfo->ipi_lastlow;
 410         } else {
 411                 first = V_ipport_firstauto;     /* sysctl */
 412                 last  = V_ipport_lastauto;
 413                 lastport = &pcbinfo->ipi_lastport;
 414         }
 415         /*
 416          * For UDP(-Lite), use random port allocation as long as the user
 417          * allows it.  For TCP (and as of yet unknown) connections,
 418          * use random port allocation only if the user allows it AND
 419          * ipport_tick() allows it.
 420          */
 421         if (V_ipport_randomized &&
 422                 (!V_ipport_stoprandom || pcbinfo == &V_udbinfo ||
 423                 pcbinfo == &V_ulitecbinfo))
 424                 dorandom = 1;
 425         else
 426                 dorandom = 0;
 427         /*
 428          * It makes no sense to do random port allocation if
 429          * we have the only port available.
 430          */
 431         if (first == last)
 432                 dorandom = 0;
 433         /* Make sure to not include UDP(-Lite) packets in the count. */
 434         if (pcbinfo != &V_udbinfo || pcbinfo != &V_ulitecbinfo)
 435                 V_ipport_tcpallocs++;
 436         /*
 437          * Instead of having two loops further down counting up or down
 438          * make sure that first is always <= last and go with only one
 439          * code path implementing all logic.
 440          */
 441         if (first > last) {
 442                 aux = first;
 443                 first = last;
 444                 last = aux;
 445         }
 446
 447 #ifdef INET
 448         /* Make the compiler happy. */
 449         laddr.s_addr = 0;
 450         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
 451                 KASSERT(laddrp != NULL, ("%s: laddrp NULL for v4 inp %p",
 452                     __func__, inp));
 453                 laddr = *laddrp;
 454         }
 455 #endif
 456         tmpinp = NULL;  /* Make compiler happy. */
 457         lport = *lportp;
 458
 459         if (dorandom)
 460                 *lastport = first + (arc4random() % (last - first));
 461
 462         count = last - first;
 463
 464         do {
 465                 if (count-- < 0)        /* completely used? */
 466                         return (EADDRNOTAVAIL);
 467                 ++*lastport;
 468                 if (*lastport < first || *lastport > last)
 469                         *lastport = first;
 470                 lport = htons(*lastport);
 471
 472 #ifdef INET6
 473                 if ((inp->inp_vflag & INP_IPV6) != 0)
 474                         tmpinp = in6_pcblookup_local(pcbinfo,
 475                             &inp->in6p_laddr, lport, lookupflags, cred);
 476 #endif
 477 #if defined(INET) && defined(INET6)
 478                 else
 479 #endif
 480 #ifdef INET
 481                         tmpinp = in_pcblookup_local(pcbinfo, laddr,
 482                             lport, lookupflags, cred);
 483 #endif
 484         } while (tmpinp != NULL);
 485
 486 #ifdef INET
 487         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4)
 488                 laddrp->s_addr = laddr.s_addr;
 489 #endif
 490         *lportp = lport;
 491
 492         return (0);
 493 }
 494
 495 /*
 496  * Return cached socket options.
 497  */
 498 short
 499 inp_so_options(const struct inpcb *inp)
 500 {
 501    short so_options;
 502
 503    so_options = 0;
 504
 505    if ((inp->inp_flags2 & INP_REUSEPORT) != 0)
 506            so_options |= SO_REUSEPORT;
 507    if ((inp->inp_flags2 & INP_REUSEADDR) != 0)
 508            so_options |= SO_REUSEADDR;
 509    return (so_options);
 510 }
 511 #endif /* INET || INET6 */
 512
 513 /*
 514  * Check if a new BINDMULTI socket is allowed to be created.
 515  *
 516  * ni points to the new inp.
 517  * oi points to the exisitng inp.
 518  *
 519  * This checks whether the existing inp also has BINDMULTI and
 520  * whether the credentials match.
 521  */
 522 int
 523 in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
 524 {
 525         /* Check permissions match */
 526         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 527             (ni->inp_cred->cr_uid !=
 528             oi->inp_cred->cr_uid))
 529                 return (0);
 530
 531         /* Check the existing inp has BINDMULTI set */
 532         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 533             ((oi->inp_flags2 & INP_BINDMULTI) == 0))
 534                 return (0);
 535
 536         /*
 537          * We're okay - either INP_BINDMULTI isn't set on ni, or
 538          * it is and it matches the checks.
 539          */
 540         return (1);
 541 }
 542
 543 #ifdef INET
 544 /*
 545  * Set up a bind operation on a PCB, performing port allocation
 546  * as required, but do not actually modify the PCB. Callers can
 547  * either complete the bind by setting inp_laddr/inp_lport and
 548  * calling in_pcbinshash(), or they can just use the resulting
 549  * port and address to authorise the sending of a once-off packet.
 550  *
 551  * On error, the values of *laddrp and *lportp are not changed.
 552  */
 553 int
 554 in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
 555     u_short *lportp, struct ucred *cred)
 556 {
 557         struct socket *so = inp->inp_socket;
 558         struct sockaddr_in *sin;
 559         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 560         struct in_addr laddr;
 561         u_short lport = 0;
 562         int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
 563         int error;
 564
 565         /*
 566          * No state changes, so read locks are sufficient here.
 567          */
 568         INP_LOCK_ASSERT(inp);
 569         INP_HASH_LOCK_ASSERT(pcbinfo);
 570
 571         if (TAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */
 572                 return (EADDRNOTAVAIL);
 573         laddr.s_addr = *laddrp;
 574         if (nam != NULL && laddr.s_addr != INADDR_ANY)
 575                 return (EINVAL);
 576         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) == 0)
 577                 lookupflags = INPLOOKUP_WILDCARD;
 578         if (nam == NULL) {
 579                 if ((error = prison_local_ip4(cred, &laddr)) != 0)
 580                         return (error);
 581         } else {
 582                 sin = (struct sockaddr_in *)nam;
 583                 if (nam->sa_len != sizeof (*sin))
 584                         return (EINVAL);
 585 #ifdef notdef
 586                 /*
 587                  * We should check the family, but old programs
 588                  * incorrectly fail to initialize it.
 589                  */
 590                 if (sin->sin_family != AF_INET)
 591                         return (EAFNOSUPPORT);
 592 #endif
 593                 error = prison_local_ip4(cred, &sin->sin_addr);
 594                 if (error)
 595                         return (error);
 596                 if (sin->sin_port != *lportp) {
 597                         /* Don't allow the port to change. */
 598                         if (*lportp != 0)
 599                                 return (EINVAL);
 600                         lport = sin->sin_port;
 601                 }
 602                 /* NB: lport is left as 0 if the port isn't being changed. */
 603                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 604                         /*
 605                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 606                          * allow complete duplication of binding if
 607                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 608                          * and a multicast address is bound on both
 609                          * new and duplicated sockets.
 610                          */
 611                         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
 612                                 reuseport = SO_REUSEADDR|SO_REUSEPORT;
 613                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 614                         sin->sin_port = 0;              /* yech... */
 615                         bzero(&sin->sin_zero, sizeof(sin->sin_zero));
 616                         /*
 617                          * Is the address a local IP address?
 618                          * If INP_BINDANY is set, then the socket may be bound
 619                          * to any endpoint address, local or not.
 620                          */
 621                         if ((inp->inp_flags & INP_BINDANY) == 0 &&
 622                             ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
 623                                 return (EADDRNOTAVAIL);
 624                 }
 625                 laddr = sin->sin_addr;
 626                 if (lport) {
 627                         struct inpcb *t;
 628                         struct tcptw *tw;
 629
 630                         /* GROSS */
 631                         if (ntohs(lport) <= V_ipport_reservedhigh &&
 632                             ntohs(lport) >= V_ipport_reservedlow &&
 633                             priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT,
 634                             0))
 635                                 return (EACCES);
 636                         if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 637                             priv_check_cred(inp->inp_cred,
 638                             PRIV_NETINET_REUSEPORT, 0) != 0) {
 639                                 t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 640                                     lport, INPLOOKUP_WILDCARD, cred);
 641         /*
 642          * XXX
 643          * This entire block sorely needs a rewrite.
 644          */
 645                                 if (t &&
 646                                     ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 647                                     ((t->inp_flags & INP_TIMEWAIT) == 0) &&
 648                                     (so->so_type != SOCK_STREAM ||
 649                                      ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
 650                                     (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
 651                                      ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
 652                                      (t->inp_flags2 & INP_REUSEPORT) == 0) &&
 653                                     (inp->inp_cred->cr_uid !=
 654                                      t->inp_cred->cr_uid))
 655                                         return (EADDRINUSE);
 656
 657                                 /*
 658                                  * If the socket is a BINDMULTI socket, then
 659                                  * the credentials need to match and the
 660                                  * original socket also has to have been bound
 661                                  * with BINDMULTI.
 662                                  */
 663                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 664                                         return (EADDRINUSE);
 665                         }
 666                         t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 667                             lport, lookupflags, cred);
 668                         if (t && (t->inp_flags & INP_TIMEWAIT)) {
 669                                 /*
 670                                  * XXXRW: If an incpb has had its timewait
 671                                  * state recycled, we treat the address as
 672                                  * being in use (for now).  This is better
 673                                  * than a panic, but not desirable.
 674                                  */
 675                                 tw = intotw(t);
 676                                 if (tw == NULL ||
 677                                     (reuseport & tw->tw_so_options) == 0)
 678                                         return (EADDRINUSE);
 679                         } else if (t &&
 680                             ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 681                             (reuseport & inp_so_options(t)) == 0) {
 682 #ifdef INET6
 683                                 if (ntohl(sin->sin_addr.s_addr) !=
 684                                     INADDR_ANY ||
 685                                     ntohl(t->inp_laddr.s_addr) !=
 686                                     INADDR_ANY ||
 687                                     (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
 688                                     (t->inp_vflag & INP_IPV6PROTO) == 0)
 689 #endif
 690                                 return (EADDRINUSE);
 691                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 692                                         return (EADDRINUSE);
 693                         }
 694                 }
 695         }
 696         if (*lportp != 0)
 697                 lport = *lportp;
 698         if (lport == 0) {
 699                 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
 700                 if (error != 0)
 701                         return (error);
 702
 703         }
 704         *laddrp = laddr.s_addr;
 705         *lportp = lport;
 706         return (0);
 707 }
 708
 709 /*
 710  * Connect from a socket to a specified address.
 711  * Both address and port must be specified in argument sin.
 712  * If don't have a local address for this socket yet,
 713  * then pick one.
 714  */
 715 int
 716 in_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
 717     struct ucred *cred, struct mbuf *m)
 718 {
 719         u_short lport, fport;
 720         in_addr_t laddr, faddr;
 721         int anonport, error;
 722
 723         INP_WLOCK_ASSERT(inp);
 724         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 725
 726         lport = inp->inp_lport;
 727         laddr = inp->inp_laddr.s_addr;
 728         anonport = (lport == 0);
 729         error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport,
 730             NULL, cred);
 731         if (error)
 732                 return (error);
 733
 734         /* Do the initial binding of the local address if required. */
 735         if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
 736                 inp->inp_lport = lport;
 737                 inp->inp_laddr.s_addr = laddr;
 738                 if (in_pcbinshash(inp) != 0) {
 739                         inp->inp_laddr.s_addr = INADDR_ANY;
 740                         inp->inp_lport = 0;
 741                         return (EAGAIN);
 742                 }
 743         }
 744
 745         /* Commit the remaining changes. */
 746         inp->inp_lport = lport;
 747         inp->inp_laddr.s_addr = laddr;
 748         inp->inp_faddr.s_addr = faddr;
 749         inp->inp_fport = fport;
 750         in_pcbrehash_mbuf(inp, m);
 751
 752         if (anonport)
 753                 inp->inp_flags |= INP_ANONPORT;
 754         return (0);
 755 }
 756
 757 int
 758 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 759 {
 760
 761         return (in_pcbconnect_mbuf(inp, nam, cred, NULL));
 762 }
 763
 764 /*
 765  * Do proper source address selection on an unbound socket in case
 766  * of connect. Take jails into account as well.
 767  */
 768 int
 769 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
 770     struct ucred *cred)
 771 {
 772         struct ifaddr *ifa;
 773         struct sockaddr *sa;
 774         struct sockaddr_in *sin;
 775         struct route sro;
 776         int error;
 777
 778         KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
 779
 780         /*
 781          * Bypass source address selection and use the primary jail IP
 782          * if requested.
 783          */
 784         if (cred != NULL && !prison_saddrsel_ip4(cred, laddr))
 785                 return (0);
 786
 787         error = 0;
 788         bzero(&sro, sizeof(sro));
 789
 790         sin = (struct sockaddr_in *)&sro.ro_dst;
 791         sin->sin_family = AF_INET;
 792         sin->sin_len = sizeof(struct sockaddr_in);
 793         sin->sin_addr.s_addr = faddr->s_addr;
 794
 795         /*
 796          * If route is known our src addr is taken from the i/f,
 797          * else punt.
 798          *
 799          * Find out route to destination.
 800          */
 801         if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
 802                 in_rtalloc_ign(&sro, 0, inp->inp_inc.inc_fibnum);
 803
 804         /*
 805          * If we found a route, use the address corresponding to
 806          * the outgoing interface.
 807          *
 808          * Otherwise assume faddr is reachable on a directly connected
 809          * network and try to find a corresponding interface to take
 810          * the source address from.
 811          */
 812         if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) {
 813                 struct in_ifaddr *ia;
 814                 struct ifnet *ifp;
 815
 816                 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
 817                                         inp->inp_socket->so_fibnum));
 818                 if (ia == NULL)
 819                         ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
 820                                                 inp->inp_socket->so_fibnum));
 821                 if (ia == NULL) {
 822                         error = ENETUNREACH;
 823                         goto done;
 824                 }
 825
 826                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 827                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 828                         ifa_free(&ia->ia_ifa);
 829                         goto done;
 830                 }
 831
 832                 ifp = ia->ia_ifp;
 833                 ifa_free(&ia->ia_ifa);
 834                 ia = NULL;
 835                 IF_ADDR_RLOCK(ifp);
 836                 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 837
 838                         sa = ifa->ifa_addr;
 839                         if (sa->sa_family != AF_INET)
 840                                 continue;
 841                         sin = (struct sockaddr_in *)sa;
 842                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 843                                 ia = (struct in_ifaddr *)ifa;
 844                                 break;
 845                         }
 846                 }
 847                 if (ia != NULL) {
 848                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 849                         IF_ADDR_RUNLOCK(ifp);
 850                         goto done;
 851                 }
 852                 IF_ADDR_RUNLOCK(ifp);
 853
 854                 /* 3. As a last resort return the 'default' jail address. */
 855                 error = prison_get_ip4(cred, laddr);
 856                 goto done;
 857         }
 858
 859         /*
 860          * If the outgoing interface on the route found is not
 861          * a loopback interface, use the address from that interface.
 862          * In case of jails do those three steps:
 863          * 1. check if the interface address belongs to the jail. If so use it.
 864          * 2. check if we have any address on the outgoing interface
 865          *    belonging to this jail. If so use it.
 866          * 3. as a last resort return the 'default' jail address.
 867          */
 868         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) == 0) {
 869                 struct in_ifaddr *ia;
 870                 struct ifnet *ifp;
 871
 872                 /* If not jailed, use the default returned. */
 873                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 874                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 875                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 876                         goto done;
 877                 }
 878
 879                 /* Jailed. */
 880                 /* 1. Check if the iface address belongs to the jail. */
 881                 sin = (struct sockaddr_in *)sro.ro_rt->rt_ifa->ifa_addr;
 882                 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 883                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 884                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 885                         goto done;
 886                 }
 887
 888                 /*
 889                  * 2. Check if we have any address on the outgoing interface
 890                  *    belonging to this jail.
 891                  */
 892                 ia = NULL;
 893                 ifp = sro.ro_rt->rt_ifp;
 894                 IF_ADDR_RLOCK(ifp);
 895                 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 896                         sa = ifa->ifa_addr;
 897                         if (sa->sa_family != AF_INET)
 898                                 continue;
 899                         sin = (struct sockaddr_in *)sa;
 900                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 901                                 ia = (struct in_ifaddr *)ifa;
 902                                 break;
 903                         }
 904                 }
 905                 if (ia != NULL) {
 906                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 907                         IF_ADDR_RUNLOCK(ifp);
 908                         goto done;
 909                 }
 910                 IF_ADDR_RUNLOCK(ifp);
 911
 912                 /* 3. As a last resort return the 'default' jail address. */
 913                 error = prison_get_ip4(cred, laddr);
 914                 goto done;
 915         }
 916
 917         /*
 918          * The outgoing interface is marked with 'loopback net', so a route
 919          * to ourselves is here.
 920          * Try to find the interface of the destination address and then
 921          * take the address from there. That interface is not necessarily
 922          * a loopback interface.
 923          * In case of jails, check that it is an address of the jail
 924          * and if we cannot find, fall back to the 'default' jail address.
 925          */
 926         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
 927                 struct sockaddr_in sain;
 928                 struct in_ifaddr *ia;
 929
 930                 bzero(&sain, sizeof(struct sockaddr_in));
 931                 sain.sin_family = AF_INET;
 932                 sain.sin_len = sizeof(struct sockaddr_in);
 933                 sain.sin_addr.s_addr = faddr->s_addr;
 934
 935                 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&sain),
 936                                         inp->inp_socket->so_fibnum));
 937                 if (ia == NULL)
 938                         ia = ifatoia(ifa_ifwithnet(sintosa(&sain), 0,
 939                                                 inp->inp_socket->so_fibnum));
 940                 if (ia == NULL)
 941                         ia = ifatoia(ifa_ifwithaddr(sintosa(&sain)));
 942
 943                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 944                         if (ia == NULL) {
 945                                 error = ENETUNREACH;
 946                                 goto done;
 947                         }
 948                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 949                         ifa_free(&ia->ia_ifa);
 950                         goto done;
 951                 }
 952
 953                 /* Jailed. */
 954                 if (ia != NULL) {
 955                         struct ifnet *ifp;
 956
 957                         ifp = ia->ia_ifp;
 958                         ifa_free(&ia->ia_ifa);
 959                         ia = NULL;
 960                         IF_ADDR_RLOCK(ifp);
 961                         TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 962
 963                                 sa = ifa->ifa_addr;
 964                                 if (sa->sa_family != AF_INET)
 965                                         continue;
 966                                 sin = (struct sockaddr_in *)sa;
 967                                 if (prison_check_ip4(cred,
 968                                     &sin->sin_addr) == 0) {
 969                                         ia = (struct in_ifaddr *)ifa;
 970                                         break;
 971                                 }
 972                         }
 973                         if (ia != NULL) {
 974                                 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 975                                 IF_ADDR_RUNLOCK(ifp);
 976                                 goto done;
 977                         }
 978                         IF_ADDR_RUNLOCK(ifp);
 979                 }
 980
 981                 /* 3. As a last resort return the 'default' jail address. */
 982                 error = prison_get_ip4(cred, laddr);
 983                 goto done;
 984         }
 985
 986 done:
 987         if (sro.ro_rt != NULL)
 988                 RTFREE(sro.ro_rt);
 989         return (error);
 990 }
 991
 992 /*
 993  * Set up for a connect from a socket to the specified address.
 994  * On entry, *laddrp and *lportp should contain the current local
 995  * address and port for the PCB; these are updated to the values
 996  * that should be placed in inp_laddr and inp_lport to complete
 997  * the connect.
 998  *
 999  * On success, *faddrp and *fportp will be set to the remote address
1000  * and port. These are not updated in the error case.
1001  *
1002  * If the operation fails because the connection already exists,
1003  * *oinpp will be set to the PCB of that connection so that the
1004  * caller can decide to override it. In all other cases, *oinpp
1005  * is set to NULL.
1006  */
1007 int
1008 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
1009     in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1010     struct inpcb **oinpp, struct ucred *cred)
1011 {
1012         struct rm_priotracker in_ifa_tracker;
1013         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1014         struct in_ifaddr *ia;
1015         struct inpcb *oinp;
1016         struct in_addr laddr, faddr;
1017         u_short lport, fport;
1018         int error;
1019
1020         /*
1021          * Because a global state change doesn't actually occur here, a read
1022          * lock is sufficient.
1023          */
1024         INP_LOCK_ASSERT(inp);
1025         INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1026
1027         if (oinpp != NULL)
1028                 *oinpp = NULL;
1029         if (nam->sa_len != sizeof (*sin))
1030                 return (EINVAL);
1031         if (sin->sin_family != AF_INET)
1032                 return (EAFNOSUPPORT);
1033         if (sin->sin_port == 0)
1034                 return (EADDRNOTAVAIL);
1035         laddr.s_addr = *laddrp;
1036         lport = *lportp;
1037         faddr = sin->sin_addr;
1038         fport = sin->sin_port;
1039
1040         if (!TAILQ_EMPTY(&V_in_ifaddrhead)) {
1041                 /*
1042                  * If the destination address is INADDR_ANY,
1043                  * use the primary local address.
1044                  * If the supplied address is INADDR_BROADCAST,
1045                  * and the primary interface supports broadcast,
1046                  * choose the broadcast address for that interface.
1047                  */
1048                 if (faddr.s_addr == INADDR_ANY) {
1049                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1050                         faddr =
1051                             IA_SIN(TAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1052                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1053                         if (cred != NULL &&
1054                             (error = prison_get_ip4(cred, &faddr)) != 0)
1055                                 return (error);
1056                 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1057                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1058                         if (TAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1059                             IFF_BROADCAST)
1060                                 faddr = satosin(&TAILQ_FIRST(
1061                                     &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1062                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1063                 }
1064         }
1065         if (laddr.s_addr == INADDR_ANY) {
1066                 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1067                 /*
1068                  * If the destination address is multicast and an outgoing
1069                  * interface has been set as a multicast option, prefer the
1070                  * address of that interface as our source address.
1071                  */
1072                 if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1073                     inp->inp_moptions != NULL) {
1074                         struct ip_moptions *imo;
1075                         struct ifnet *ifp;
1076
1077                         imo = inp->inp_moptions;
1078                         if (imo->imo_multicast_ifp != NULL) {
1079                                 ifp = imo->imo_multicast_ifp;
1080                                 IN_IFADDR_RLOCK(&in_ifa_tracker);
1081                                 TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1082                                         if ((ia->ia_ifp == ifp) &&
1083                                             (cred == NULL ||
1084                                             prison_check_ip4(cred,
1085                                             &ia->ia_addr.sin_addr) == 0))
1086                                                 break;
1087                                 }
1088                                 if (ia == NULL)
1089                                         error = EADDRNOTAVAIL;
1090                                 else {
1091                                         laddr = ia->ia_addr.sin_addr;
1092                                         error = 0;
1093                                 }
1094                                 IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1095                         }
1096                 }
1097                 if (error)
1098                         return (error);
1099         }
1100         oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, fport,
1101             laddr, lport, 0, NULL);
1102         if (oinp != NULL) {
1103                 if (oinpp != NULL)
1104                         *oinpp = oinp;
1105                 return (EADDRINUSE);
1106         }
1107         if (lport == 0) {
1108                 error = in_pcbbind_setup(inp, NULL, &laddr.s_addr, &lport,
1109                     cred);
1110                 if (error)
1111                         return (error);
1112         }
1113         *laddrp = laddr.s_addr;
1114         *lportp = lport;
1115         *faddrp = faddr.s_addr;
1116         *fportp = fport;
1117         return (0);
1118 }
1119
1120 void
1121 in_pcbdisconnect(struct inpcb *inp)
1122 {
1123
1124         INP_WLOCK_ASSERT(inp);
1125         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1126
1127         inp->inp_faddr.s_addr = INADDR_ANY;
1128         inp->inp_fport = 0;
1129         in_pcbrehash(inp);
1130 }
1131 #endif /* INET */
1132
1133 /*
1134  * in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
1135  * For most protocols, this will be invoked immediately prior to calling
1136  * in_pcbfree().  However, with TCP the inpcb may significantly outlive the
1137  * socket, in which case in_pcbfree() is deferred.
1138  */
1139 void
1140 in_pcbdetach(struct inpcb *inp)
1141 {
1142
1143         KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1144
1145 #ifdef RATELIMIT
1146         if (inp->inp_snd_tag != NULL)
1147                 in_pcbdetach_txrtlmt(inp);
1148 #endif
1149         inp->inp_socket->so_pcb = NULL;
1150         inp->inp_socket = NULL;
1151 }
1152
1153 /*
1154  * in_pcbref() bumps the reference count on an inpcb in order to maintain
1155  * stability of an inpcb pointer despite the inpcb lock being released.  This
1156  * is used in TCP when the inpcbinfo lock needs to be acquired or upgraded,
1157  * but where the inpcb lock may already held, or when acquiring a reference
1158  * via a pcbgroup.
1159  *
1160  * in_pcbref() should be used only to provide brief memory stability, and
1161  * must always be followed by a call to INP_WLOCK() and in_pcbrele() to
1162  * garbage collect the inpcb if it has been in_pcbfree()'d from another
1163  * context.  Until in_pcbrele() has returned that the inpcb is still valid,
1164  * lock and rele are the *only* safe operations that may be performed on the
1165  * inpcb.
1166  *
1167  * While the inpcb will not be freed, releasing the inpcb lock means that the
1168  * connection's state may change, so the caller should be careful to
1169  * revalidate any cached state on reacquiring the lock.  Drop the reference
1170  * using in_pcbrele().
1171  */
1172 void
1173 in_pcbref(struct inpcb *inp)
1174 {
1175
1176         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1177
1178         refcount_acquire(&inp->inp_refcount);
1179 }
1180
1181 /*
1182  * Drop a refcount on an inpcb elevated using in_pcbref(); because a call to
1183  * in_pcbfree() may have been made between in_pcbref() and in_pcbrele(), we
1184  * return a flag indicating whether or not the inpcb remains valid.  If it is
1185  * valid, we return with the inpcb lock held.
1186  *
1187  * Notice that, unlike in_pcbref(), the inpcb lock must be held to drop a
1188  * reference on an inpcb.  Historically more work was done here (actually, in
1189  * in_pcbfree_internal()) but has been moved to in_pcbfree() to avoid the
1190  * need for the pcbinfo lock in in_pcbrele().  Deferring the free is entirely
1191  * about memory stability (and continued use of the write lock).
1192  */
1193 int
1194 in_pcbrele_rlocked(struct inpcb *inp)
1195 {
1196         struct inpcbinfo *pcbinfo;
1197
1198         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1199
1200         INP_RLOCK_ASSERT(inp);
1201
1202         if (refcount_release(&inp->inp_refcount) == 0) {
1203                 /*
1204                  * If the inpcb has been freed, let the caller know, even if
1205                  * this isn't the last reference.
1206                  */
1207                 if (inp->inp_flags2 & INP_FREED) {
1208                         INP_RUNLOCK(inp);
1209                         return (1);
1210                 }
1211                 return (0);
1212         }
1213
1214         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1215
1216         INP_RUNLOCK(inp);
1217         pcbinfo = inp->inp_pcbinfo;
1218         uma_zfree(pcbinfo->ipi_zone, inp);
1219         return (1);
1220 }
1221
1222 int
1223 in_pcbrele_wlocked(struct inpcb *inp)
1224 {
1225         struct inpcbinfo *pcbinfo;
1226
1227         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1228
1229         INP_WLOCK_ASSERT(inp);
1230
1231         if (refcount_release(&inp->inp_refcount) == 0) {
1232                 /*
1233                  * If the inpcb has been freed, let the caller know, even if
1234                  * this isn't the last reference.
1235                  */
1236                 if (inp->inp_flags2 & INP_FREED) {
1237                         INP_WUNLOCK(inp);
1238                         return (1);
1239                 }
1240                 return (0);
1241         }
1242
1243         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1244
1245         INP_WUNLOCK(inp);
1246         pcbinfo = inp->inp_pcbinfo;
1247         uma_zfree(pcbinfo->ipi_zone, inp);
1248         return (1);
1249 }
1250
1251 /*
1252  * Temporary wrapper.
1253  */
1254 int
1255 in_pcbrele(struct inpcb *inp)
1256 {
1257
1258         return (in_pcbrele_wlocked(inp));
1259 }
1260
1261 /*
1262  * Unconditionally schedule an inpcb to be freed by decrementing its
1263  * reference count, which should occur only after the inpcb has been detached
1264  * from its socket.  If another thread holds a temporary reference (acquired
1265  * using in_pcbref()) then the free is deferred until that reference is
1266  * released using in_pcbrele(), but the inpcb is still unlocked.  Almost all
1267  * work, including removal from global lists, is done in this context, where
1268  * the pcbinfo lock is held.
1269  */
1270 void
1271 in_pcbfree(struct inpcb *inp)
1272 {
1273         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1274
1275         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1276
1277 #ifdef INVARIANTS
1278         if (pcbinfo == &V_tcbinfo) {
1279                 INP_INFO_LOCK_ASSERT(pcbinfo);
1280         } else {
1281                 INP_INFO_WLOCK_ASSERT(pcbinfo);
1282         }
1283 #endif
1284         INP_WLOCK_ASSERT(inp);
1285
1286         /* XXXRW: Do as much as possible here. */
1287 #ifdef IPSEC
1288         if (inp->inp_sp != NULL)
1289                 ipsec_delete_pcbpolicy(inp);
1290 #endif
1291         INP_LIST_WLOCK(pcbinfo);
1292         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1293         in_pcbremlists(inp);
1294         INP_LIST_WUNLOCK(pcbinfo);
1295 #ifdef INET6
1296         if (inp->inp_vflag & INP_IPV6PROTO) {
1297                 ip6_freepcbopts(inp->in6p_outputopts);
1298                 if (inp->in6p_moptions != NULL)
1299                         ip6_freemoptions(inp->in6p_moptions);
1300         }
1301 #endif
1302         if (inp->inp_options)
1303                 (void)m_free(inp->inp_options);
1304 #ifdef INET
1305         if (inp->inp_moptions != NULL)
1306                 inp_freemoptions(inp->inp_moptions);
1307 #endif
1308         RO_RTFREE(&inp->inp_route);
1309         if (inp->inp_route.ro_lle)
1310                 LLE_FREE(inp->inp_route.ro_lle);        /* zeros ro_lle */
1311
1312         inp->inp_vflag = 0;
1313         inp->inp_flags2 |= INP_FREED;
1314         crfree(inp->inp_cred);
1315 #ifdef MAC
1316         mac_inpcb_destroy(inp);
1317 #endif
1318         if (!in_pcbrele_wlocked(inp))
1319                 INP_WUNLOCK(inp);
1320 }
1321
1322 /*
1323  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1324  * port reservation, and preventing it from being returned by inpcb lookups.
1325  *
1326  * It is used by TCP to mark an inpcb as unused and avoid future packet
1327  * delivery or event notification when a socket remains open but TCP has
1328  * closed.  This might occur as a result of a shutdown()-initiated TCP close
1329  * or a RST on the wire, and allows the port binding to be reused while still
1330  * maintaining the invariant that so_pcb always points to a valid inpcb until
1331  * in_pcbdetach().
1332  *
1333  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1334  * in_pcbnotifyall() and in_pcbpurgeif0()?
1335  */
1336 void
1337 in_pcbdrop(struct inpcb *inp)
1338 {
1339
1340         INP_WLOCK_ASSERT(inp);
1341
1342         /*
1343          * XXXRW: Possibly we should protect the setting of INP_DROPPED with
1344          * the hash lock...?
1345          */
1346         inp->inp_flags |= INP_DROPPED;
1347         if (inp->inp_flags & INP_INHASHLIST) {
1348                 struct inpcbport *phd = inp->inp_phd;
1349
1350                 INP_HASH_WLOCK(inp->inp_pcbinfo);
1351                 LIST_REMOVE(inp, inp_hash);
1352                 LIST_REMOVE(inp, inp_portlist);
1353                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
1354                         LIST_REMOVE(phd, phd_hash);
1355                         free(phd, M_PCB);
1356                 }
1357                 INP_HASH_WUNLOCK(inp->inp_pcbinfo);
1358                 inp->inp_flags &= ~INP_INHASHLIST;
1359 #ifdef PCBGROUP
1360                 in_pcbgroup_remove(inp);
1361 #endif
1362         }
1363 }
1364
1365 #ifdef INET
1366 /*
1367  * Common routines to return the socket addresses associated with inpcbs.
1368  */
1369 struct sockaddr *
1370 in_sockaddr(in_port_t port, struct in_addr *addr_p)
1371 {
1372         struct sockaddr_in *sin;
1373
1374         sin = malloc(sizeof *sin, M_SONAME,
1375                 M_WAITOK | M_ZERO);
1376         sin->sin_family = AF_INET;
1377         sin->sin_len = sizeof(*sin);
1378         sin->sin_addr = *addr_p;
1379         sin->sin_port = port;
1380
1381         return (struct sockaddr *)sin;
1382 }
1383
1384 int
1385 in_getsockaddr(struct socket *so, struct sockaddr **nam)
1386 {
1387         struct inpcb *inp;
1388         struct in_addr addr;
1389         in_port_t port;
1390
1391         inp = sotoinpcb(so);
1392         KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1393
1394         INP_RLOCK(inp);
1395         port = inp->inp_lport;
1396         addr = inp->inp_laddr;
1397         INP_RUNLOCK(inp);
1398
1399         *nam = in_sockaddr(port, &addr);
1400         return 0;
1401 }
1402
1403 int
1404 in_getpeeraddr(struct socket *so, struct sockaddr **nam)
1405 {
1406         struct inpcb *inp;
1407         struct in_addr addr;
1408         in_port_t port;
1409
1410         inp = sotoinpcb(so);
1411         KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1412
1413         INP_RLOCK(inp);
1414         port = inp->inp_fport;
1415         addr = inp->inp_faddr;
1416         INP_RUNLOCK(inp);
1417
1418         *nam = in_sockaddr(port, &addr);
1419         return 0;
1420 }
1421
1422 void
1423 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
1424     struct inpcb *(*notify)(struct inpcb *, int))
1425 {
1426         struct inpcb *inp, *inp_temp;
1427
1428         INP_INFO_WLOCK(pcbinfo);
1429         LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
1430                 INP_WLOCK(inp);
1431 #ifdef INET6
1432                 if ((inp->inp_vflag & INP_IPV4) == 0) {
1433                         INP_WUNLOCK(inp);
1434                         continue;
1435                 }
1436 #endif
1437                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1438                     inp->inp_socket == NULL) {
1439                         INP_WUNLOCK(inp);
1440                         continue;
1441                 }
1442                 if ((*notify)(inp, errno))
1443                         INP_WUNLOCK(inp);
1444         }
1445         INP_INFO_WUNLOCK(pcbinfo);
1446 }
1447
1448 void
1449 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1450 {
1451         struct inpcb *inp;
1452         struct ip_moptions *imo;
1453         int i, gap;
1454
1455         INP_INFO_WLOCK(pcbinfo);
1456         LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1457                 INP_WLOCK(inp);
1458                 imo = inp->inp_moptions;
1459                 if ((inp->inp_vflag & INP_IPV4) &&
1460                     imo != NULL) {
1461                         /*
1462                          * Unselect the outgoing interface if it is being
1463                          * detached.
1464                          */
1465                         if (imo->imo_multicast_ifp == ifp)
1466                                 imo->imo_multicast_ifp = NULL;
1467
1468                         /*
1469                          * Drop multicast group membership if we joined
1470                          * through the interface being detached.
1471                          */
1472                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1473                             i++) {
1474                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1475                                         in_delmulti(imo->imo_membership[i]);
1476                                         gap++;
1477                                 } else if (gap != 0)
1478                                         imo->imo_membership[i - gap] =
1479                                             imo->imo_membership[i];
1480                         }
1481                         imo->imo_num_memberships -= gap;
1482                 }
1483                 INP_WUNLOCK(inp);
1484         }
1485         INP_INFO_WUNLOCK(pcbinfo);
1486 }
1487
1488 /*
1489  * Lookup a PCB based on the local address and port.  Caller must hold the
1490  * hash lock.  No inpcb locks or references are acquired.
1491  */
1492 #define INP_LOOKUP_MAPPED_PCB_COST      3
1493 struct inpcb *
1494 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1495     u_short lport, int lookupflags, struct ucred *cred)
1496 {
1497         struct inpcb *inp;
1498 #ifdef INET6
1499         int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1500 #else
1501         int matchwild = 3;
1502 #endif
1503         int wildcard;
1504
1505         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1506             ("%s: invalid lookup flags %d", __func__, lookupflags));
1507
1508         INP_HASH_LOCK_ASSERT(pcbinfo);
1509
1510         if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1511                 struct inpcbhead *head;
1512                 /*
1513                  * Look for an unconnected (wildcard foreign addr) PCB that
1514                  * matches the local address and port we're looking for.
1515                  */
1516                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1517                     0, pcbinfo->ipi_hashmask)];
1518                 LIST_FOREACH(inp, head, inp_hash) {
1519 #ifdef INET6
1520                         /* XXX inp locking */
1521                         if ((inp->inp_vflag & INP_IPV4) == 0)
1522                                 continue;
1523 #endif
1524                         if (inp->inp_faddr.s_addr == INADDR_ANY &&
1525                             inp->inp_laddr.s_addr == laddr.s_addr &&
1526                             inp->inp_lport == lport) {
1527                                 /*
1528                                  * Found?
1529                                  */
1530                                 if (cred == NULL ||
1531                                     prison_equal_ip4(cred->cr_prison,
1532                                         inp->inp_cred->cr_prison))
1533                                         return (inp);
1534                         }
1535                 }
1536                 /*
1537                  * Not found.
1538                  */
1539                 return (NULL);
1540         } else {
1541                 struct inpcbporthead *porthash;
1542                 struct inpcbport *phd;
1543                 struct inpcb *match = NULL;
1544                 /*
1545                  * Best fit PCB lookup.
1546                  *
1547                  * First see if this local port is in use by looking on the
1548                  * port hash list.
1549                  */
1550                 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
1551                     pcbinfo->ipi_porthashmask)];
1552                 LIST_FOREACH(phd, porthash, phd_hash) {
1553                         if (phd->phd_port == lport)
1554                                 break;
1555                 }
1556                 if (phd != NULL) {
1557                         /*
1558                          * Port is in use by one or more PCBs. Look for best
1559                          * fit.
1560                          */
1561                         LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1562                                 wildcard = 0;
1563                                 if (cred != NULL &&
1564                                     !prison_equal_ip4(inp->inp_cred->cr_prison,
1565                                         cred->cr_prison))
1566                                         continue;
1567 #ifdef INET6
1568                                 /* XXX inp locking */
1569                                 if ((inp->inp_vflag & INP_IPV4) == 0)
1570                                         continue;
1571                                 /*
1572                                  * We never select the PCB that has
1573                                  * INP_IPV6 flag and is bound to :: if
1574                                  * we have another PCB which is bound
1575                                  * to 0.0.0.0.  If a PCB has the
1576                                  * INP_IPV6 flag, then we set its cost
1577                                  * higher than IPv4 only PCBs.
1578                                  *
1579                                  * Note that the case only happens
1580                                  * when a socket is bound to ::, under
1581                                  * the condition that the use of the
1582                                  * mapped address is allowed.
1583                                  */
1584                                 if ((inp->inp_vflag & INP_IPV6) != 0)
1585                                         wildcard += INP_LOOKUP_MAPPED_PCB_COST;
1586 #endif
1587                                 if (inp->inp_faddr.s_addr != INADDR_ANY)
1588                                         wildcard++;
1589                                 if (inp->inp_laddr.s_addr != INADDR_ANY) {
1590                                         if (laddr.s_addr == INADDR_ANY)
1591                                                 wildcard++;
1592                                         else if (inp->inp_laddr.s_addr != laddr.s_addr)
1593                                                 continue;
1594                                 } else {
1595                                         if (laddr.s_addr != INADDR_ANY)
1596                                                 wildcard++;
1597                                 }
1598                                 if (wildcard < matchwild) {
1599                                         match = inp;
1600                                         matchwild = wildcard;
1601                                         if (matchwild == 0)
1602                                                 break;
1603                                 }
1604                         }
1605                 }
1606                 return (match);
1607         }
1608 }
1609 #undef INP_LOOKUP_MAPPED_PCB_COST
1610
1611 #ifdef PCBGROUP
1612 /*
1613  * Lookup PCB in hash list, using pcbgroup tables.
1614  */
1615 static struct inpcb *
1616 in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
1617     struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
1618     u_int lport_arg, int lookupflags, struct ifnet *ifp)
1619 {
1620         struct inpcbhead *head;
1621         struct inpcb *inp, *tmpinp;
1622         u_short fport = fport_arg, lport = lport_arg;
1623
1624         /*
1625          * First look for an exact match.
1626          */
1627         tmpinp = NULL;
1628         INP_GROUP_LOCK(pcbgroup);
1629         head = &pcbgroup->ipg_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
1630             pcbgroup->ipg_hashmask)];
1631         LIST_FOREACH(inp, head, inp_pcbgrouphash) {
1632 #ifdef INET6
1633                 /* XXX inp locking */
1634                 if ((inp->inp_vflag & INP_IPV4) == 0)
1635                         continue;
1636 #endif
1637                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
1638                     inp->inp_laddr.s_addr == laddr.s_addr &&
1639                     inp->inp_fport == fport &&
1640                     inp->inp_lport == lport) {
1641                         /*
1642                          * XXX We should be able to directly return
1643                          * the inp here, without any checks.
1644                          * Well unless both bound with SO_REUSEPORT?
1645                          */
1646                         if (prison_flag(inp->inp_cred, PR_IP4))
1647                                 goto found;
1648                         if (tmpinp == NULL)
1649                                 tmpinp = inp;
1650                 }
1651         }
1652         if (tmpinp != NULL) {
1653                 inp = tmpinp;
1654                 goto found;
1655         }
1656
1657 #ifdef  RSS
1658         /*
1659          * For incoming connections, we may wish to do a wildcard
1660          * match for an RSS-local socket.
1661          */
1662         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1663                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1664 #ifdef INET6
1665                 struct inpcb *local_wild_mapped = NULL;
1666 #endif
1667                 struct inpcb *jail_wild = NULL;
1668                 struct inpcbhead *head;
1669                 int injail;
1670
1671                 /*
1672                  * Order of socket selection - we always prefer jails.
1673                  *      1. jailed, non-wild.
1674                  *      2. jailed, wild.
1675                  *      3. non-jailed, non-wild.
1676                  *      4. non-jailed, wild.
1677                  */
1678
1679                 head = &pcbgroup->ipg_hashbase[INP_PCBHASH(INADDR_ANY,
1680                     lport, 0, pcbgroup->ipg_hashmask)];
1681                 LIST_FOREACH(inp, head, inp_pcbgrouphash) {
1682 #ifdef INET6
1683                         /* XXX inp locking */
1684                         if ((inp->inp_vflag & INP_IPV4) == 0)
1685                                 continue;
1686 #endif
1687                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1688                             inp->inp_lport != lport)
1689                                 continue;
1690
1691                         injail = prison_flag(inp->inp_cred, PR_IP4);
1692                         if (injail) {
1693                                 if (prison_check_ip4(inp->inp_cred,
1694                                     &laddr) != 0)
1695                                         continue;
1696                         } else {
1697                                 if (local_exact != NULL)
1698                                         continue;
1699                         }
1700
1701                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1702                                 if (injail)
1703                                         goto found;
1704                                 else
1705                                         local_exact = inp;
1706                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1707 #ifdef INET6
1708                                 /* XXX inp locking, NULL check */
1709                                 if (inp->inp_vflag & INP_IPV6PROTO)
1710                                         local_wild_mapped = inp;
1711                                 else
1712 #endif
1713                                         if (injail)
1714                                                 jail_wild = inp;
1715                                         else
1716                                                 local_wild = inp;
1717                         }
1718                 } /* LIST_FOREACH */
1719
1720                 inp = jail_wild;
1721                 if (inp == NULL)
1722                         inp = local_exact;
1723                 if (inp == NULL)
1724                         inp = local_wild;
1725 #ifdef INET6
1726                 if (inp == NULL)
1727                         inp = local_wild_mapped;
1728 #endif
1729                 if (inp != NULL)
1730                         goto found;
1731         }
1732 #endif
1733
1734         /*
1735          * Then look for a wildcard match, if requested.
1736          */
1737         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1738                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1739 #ifdef INET6
1740                 struct inpcb *local_wild_mapped = NULL;
1741 #endif
1742                 struct inpcb *jail_wild = NULL;
1743                 struct inpcbhead *head;
1744                 int injail;
1745
1746                 /*
1747                  * Order of socket selection - we always prefer jails.
1748                  *      1. jailed, non-wild.
1749                  *      2. jailed, wild.
1750                  *      3. non-jailed, non-wild.
1751                  *      4. non-jailed, wild.
1752                  */
1753                 head = &pcbinfo->ipi_wildbase[INP_PCBHASH(INADDR_ANY, lport,
1754                     0, pcbinfo->ipi_wildmask)];
1755                 LIST_FOREACH(inp, head, inp_pcbgroup_wild) {
1756 #ifdef INET6
1757                         /* XXX inp locking */
1758                         if ((inp->inp_vflag & INP_IPV4) == 0)
1759                                 continue;
1760 #endif
1761                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1762                             inp->inp_lport != lport)
1763                                 continue;
1764
1765                         injail = prison_flag(inp->inp_cred, PR_IP4);
1766                         if (injail) {
1767                                 if (prison_check_ip4(inp->inp_cred,
1768                                     &laddr) != 0)
1769                                         continue;
1770                         } else {
1771                                 if (local_exact != NULL)
1772                                         continue;
1773                         }
1774
1775                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1776                                 if (injail)
1777                                         goto found;
1778                                 else
1779                                         local_exact = inp;
1780                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1781 #ifdef INET6
1782                                 /* XXX inp locking, NULL check */
1783                                 if (inp->inp_vflag & INP_IPV6PROTO)
1784                                         local_wild_mapped = inp;
1785                                 else
1786 #endif
1787                                         if (injail)
1788                                                 jail_wild = inp;
1789                                         else
1790                                                 local_wild = inp;
1791                         }
1792                 } /* LIST_FOREACH */
1793                 inp = jail_wild;
1794                 if (inp == NULL)
1795                         inp = local_exact;
1796                 if (inp == NULL)
1797                         inp = local_wild;
1798 #ifdef INET6
1799                 if (inp == NULL)
1800                         inp = local_wild_mapped;
1801 #endif
1802                 if (inp != NULL)
1803                         goto found;
1804         } /* if (lookupflags & INPLOOKUP_WILDCARD) */
1805         INP_GROUP_UNLOCK(pcbgroup);
1806         return (NULL);
1807
1808 found:
1809         in_pcbref(inp);
1810         INP_GROUP_UNLOCK(pcbgroup);
1811         if (lookupflags & INPLOOKUP_WLOCKPCB) {
1812                 INP_WLOCK(inp);
1813                 if (in_pcbrele_wlocked(inp))
1814                         return (NULL);
1815         } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
1816                 INP_RLOCK(inp);
1817                 if (in_pcbrele_rlocked(inp))
1818                         return (NULL);
1819         } else
1820                 panic("%s: locking bug", __func__);
1821         return (inp);
1822 }
1823 #endif /* PCBGROUP */
1824
1825 /*
1826  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
1827  * that the caller has locked the hash list, and will not perform any further
1828  * locking or reference operations on either the hash list or the connection.
1829  */
1830 static struct inpcb *
1831 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1832     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
1833     struct ifnet *ifp)
1834 {
1835         struct inpcbhead *head;
1836         struct inpcb *inp, *tmpinp;
1837         u_short fport = fport_arg, lport = lport_arg;
1838
1839         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1840             ("%s: invalid lookup flags %d", __func__, lookupflags));
1841
1842         INP_HASH_LOCK_ASSERT(pcbinfo);
1843
1844         /*
1845          * First look for an exact match.
1846          */
1847         tmpinp = NULL;
1848         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
1849             pcbinfo->ipi_hashmask)];
1850         LIST_FOREACH(inp, head, inp_hash) {
1851 #ifdef INET6
1852                 /* XXX inp locking */
1853                 if ((inp->inp_vflag & INP_IPV4) == 0)
1854                         continue;
1855 #endif
1856                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
1857                     inp->inp_laddr.s_addr == laddr.s_addr &&
1858                     inp->inp_fport == fport &&
1859                     inp->inp_lport == lport) {
1860                         /*
1861                          * XXX We should be able to directly return
1862                          * the inp here, without any checks.
1863                          * Well unless both bound with SO_REUSEPORT?
1864                          */
1865                         if (prison_flag(inp->inp_cred, PR_IP4))
1866                                 return (inp);
1867                         if (tmpinp == NULL)
1868                                 tmpinp = inp;
1869                 }
1870         }
1871         if (tmpinp != NULL)
1872                 return (tmpinp);
1873
1874         /*
1875          * Then look for a wildcard match, if requested.
1876          */
1877         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1878                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1879 #ifdef INET6
1880                 struct inpcb *local_wild_mapped = NULL;
1881 #endif
1882                 struct inpcb *jail_wild = NULL;
1883                 int injail;
1884
1885                 /*
1886                  * Order of socket selection - we always prefer jails.
1887                  *      1. jailed, non-wild.
1888                  *      2. jailed, wild.
1889                  *      3. non-jailed, non-wild.
1890                  *      4. non-jailed, wild.
1891                  */
1892
1893                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1894                     0, pcbinfo->ipi_hashmask)];
1895                 LIST_FOREACH(inp, head, inp_hash) {
1896 #ifdef INET6
1897                         /* XXX inp locking */
1898                         if ((inp->inp_vflag & INP_IPV4) == 0)
1899                                 continue;
1900 #endif
1901                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1902                             inp->inp_lport != lport)
1903                                 continue;
1904
1905                         injail = prison_flag(inp->inp_cred, PR_IP4);
1906                         if (injail) {
1907                                 if (prison_check_ip4(inp->inp_cred,
1908                                     &laddr) != 0)
1909                                         continue;
1910                         } else {
1911                                 if (local_exact != NULL)
1912                                         continue;
1913                         }
1914
1915                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1916                                 if (injail)
1917                                         return (inp);
1918                                 else
1919                                         local_exact = inp;
1920                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1921 #ifdef INET6
1922                                 /* XXX inp locking, NULL check */
1923                                 if (inp->inp_vflag & INP_IPV6PROTO)
1924                                         local_wild_mapped = inp;
1925                                 else
1926 #endif
1927                                         if (injail)
1928                                                 jail_wild = inp;
1929                                         else
1930                                                 local_wild = inp;
1931                         }
1932                 } /* LIST_FOREACH */
1933                 if (jail_wild != NULL)
1934                         return (jail_wild);
1935                 if (local_exact != NULL)
1936                         return (local_exact);
1937                 if (local_wild != NULL)
1938                         return (local_wild);
1939 #ifdef INET6
1940                 if (local_wild_mapped != NULL)
1941                         return (local_wild_mapped);
1942 #endif
1943         } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
1944
1945         return (NULL);
1946 }
1947
1948 /*
1949  * Lookup PCB in hash list, using pcbinfo tables.  This variation locks the
1950  * hash list lock, and will return the inpcb locked (i.e., requires
1951  * INPLOOKUP_LOCKPCB).
1952  */
1953 static struct inpcb *
1954 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1955     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
1956     struct ifnet *ifp)
1957 {
1958         struct inpcb *inp;
1959
1960         INP_HASH_RLOCK(pcbinfo);
1961         inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
1962             (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
1963         if (inp != NULL) {
1964                 in_pcbref(inp);
1965                 INP_HASH_RUNLOCK(pcbinfo);
1966                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
1967                         INP_WLOCK(inp);
1968                         if (in_pcbrele_wlocked(inp))
1969                                 return (NULL);
1970                 } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
1971                         INP_RLOCK(inp);
1972                         if (in_pcbrele_rlocked(inp))
1973                                 return (NULL);
1974                 } else
1975                         panic("%s: locking bug", __func__);
1976         } else
1977                 INP_HASH_RUNLOCK(pcbinfo);
1978         return (inp);
1979 }
1980
1981 /*
1982  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
1983  * from which a pre-calculated hash value may be extracted.
1984  *
1985  * Possibly more of this logic should be in in_pcbgroup.c.
1986  */
1987 struct inpcb *
1988 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
1989     struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp)
1990 {
1991 #if defined(PCBGROUP) && !defined(RSS)
1992         struct inpcbgroup *pcbgroup;
1993 #endif
1994
1995         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
1996             ("%s: invalid lookup flags %d", __func__, lookupflags));
1997         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
1998             ("%s: LOCKPCB not set", __func__));
1999
2000         /*
2001          * When not using RSS, use connection groups in preference to the
2002          * reservation table when looking up 4-tuples.  When using RSS, just
2003          * use the reservation table, due to the cost of the Toeplitz hash
2004          * in software.
2005          *
2006          * XXXRW: This policy belongs in the pcbgroup code, as in principle
2007          * we could be doing RSS with a non-Toeplitz hash that is affordable
2008          * in software.
2009          */
2010 #if defined(PCBGROUP) && !defined(RSS)
2011         if (in_pcbgroup_enabled(pcbinfo)) {
2012                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2013                     fport);
2014                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2015                     laddr, lport, lookupflags, ifp));
2016         }
2017 #endif
2018         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2019             lookupflags, ifp));
2020 }
2021
2022 struct inpcb *
2023 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2024     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2025     struct ifnet *ifp, struct mbuf *m)
2026 {
2027 #ifdef PCBGROUP
2028         struct inpcbgroup *pcbgroup;
2029 #endif
2030
2031         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2032             ("%s: invalid lookup flags %d", __func__, lookupflags));
2033         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2034             ("%s: LOCKPCB not set", __func__));
2035
2036 #ifdef PCBGROUP
2037         /*
2038          * If we can use a hardware-generated hash to look up the connection
2039          * group, use that connection group to find the inpcb.  Otherwise
2040          * fall back on a software hash -- or the reservation table if we're
2041          * using RSS.
2042          *
2043          * XXXRW: As above, that policy belongs in the pcbgroup code.
2044          */
2045         if (in_pcbgroup_enabled(pcbinfo) &&
2046             !(M_HASHTYPE_TEST(m, M_HASHTYPE_NONE))) {
2047                 pcbgroup = in_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
2048                     m->m_pkthdr.flowid);
2049                 if (pcbgroup != NULL)
2050                         return (in_pcblookup_group(pcbinfo, pcbgroup, faddr,
2051                             fport, laddr, lport, lookupflags, ifp));
2052 #ifndef RSS
2053                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2054                     fport);
2055                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2056                     laddr, lport, lookupflags, ifp));
2057 #endif
2058         }
2059 #endif
2060         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2061             lookupflags, ifp));
2062 }
2063 #endif /* INET */
2064
2065 /*
2066  * Insert PCB onto various hash lists.
2067  */
2068 static int
2069 in_pcbinshash_internal(struct inpcb *inp, int do_pcbgroup_update)
2070 {
2071         struct inpcbhead *pcbhash;
2072         struct inpcbporthead *pcbporthash;
2073         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2074         struct inpcbport *phd;
2075         u_int32_t hashkey_faddr;
2076
2077         INP_WLOCK_ASSERT(inp);
2078         INP_HASH_WLOCK_ASSERT(pcbinfo);
2079
2080         KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2081             ("in_pcbinshash: INP_INHASHLIST"));
2082
2083 #ifdef INET6
2084         if (inp->inp_vflag & INP_IPV6)
2085                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2086         else
2087 #endif
2088         hashkey_faddr = inp->inp_faddr.s_addr;
2089
2090         pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2091                  inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2092
2093         pcbporthash = &pcbinfo->ipi_porthashbase[
2094             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2095
2096         /*
2097          * Go through port list and look for a head for this lport.
2098          */
2099         LIST_FOREACH(phd, pcbporthash, phd_hash) {
2100                 if (phd->phd_port == inp->inp_lport)
2101                         break;
2102         }
2103         /*
2104          * If none exists, malloc one and tack it on.
2105          */
2106         if (phd == NULL) {
2107                 phd = malloc(sizeof(struct inpcbport), M_PCB, M_NOWAIT);
2108                 if (phd == NULL) {
2109                         return (ENOBUFS); /* XXX */
2110                 }
2111                 phd->phd_port = inp->inp_lport;
2112                 LIST_INIT(&phd->phd_pcblist);
2113                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2114         }
2115         inp->inp_phd = phd;
2116         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2117         LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
2118         inp->inp_flags |= INP_INHASHLIST;
2119 #ifdef PCBGROUP
2120         if (do_pcbgroup_update)
2121                 in_pcbgroup_update(inp);
2122 #endif
2123         return (0);
2124 }
2125
2126 /*
2127  * For now, there are two public interfaces to insert an inpcb into the hash
2128  * lists -- one that does update pcbgroups, and one that doesn't.  The latter
2129  * is used only in the TCP syncache, where in_pcbinshash is called before the
2130  * full 4-tuple is set for the inpcb, and we don't want to install in the
2131  * pcbgroup until later.
2132  *
2133  * XXXRW: This seems like a misfeature.  in_pcbinshash should always update
2134  * connection groups, and partially initialised inpcbs should not be exposed
2135  * to either reservation hash tables or pcbgroups.
2136  */
2137 int
2138 in_pcbinshash(struct inpcb *inp)
2139 {
2140
2141         return (in_pcbinshash_internal(inp, 1));
2142 }
2143
2144 int
2145 in_pcbinshash_nopcbgroup(struct inpcb *inp)
2146 {
2147
2148         return (in_pcbinshash_internal(inp, 0));
2149 }
2150
2151 /*
2152  * Move PCB to the proper hash bucket when { faddr, fport } have  been
2153  * changed. NOTE: This does not handle the case of the lport changing (the
2154  * hashed port list would have to be updated as well), so the lport must
2155  * not change after in_pcbinshash() has been called.
2156  */
2157 void
2158 in_pcbrehash_mbuf(struct inpcb *inp, struct mbuf *m)
2159 {
2160         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2161         struct inpcbhead *head;
2162         u_int32_t hashkey_faddr;
2163
2164         INP_WLOCK_ASSERT(inp);
2165         INP_HASH_WLOCK_ASSERT(pcbinfo);
2166
2167         KASSERT(inp->inp_flags & INP_INHASHLIST,
2168             ("in_pcbrehash: !INP_INHASHLIST"));
2169
2170 #ifdef INET6
2171         if (inp->inp_vflag & INP_IPV6)
2172                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2173         else
2174 #endif
2175         hashkey_faddr = inp->inp_faddr.s_addr;
2176
2177         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2178                 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2179
2180         LIST_REMOVE(inp, inp_hash);
2181         LIST_INSERT_HEAD(head, inp, inp_hash);
2182
2183 #ifdef PCBGROUP
2184         if (m != NULL)
2185                 in_pcbgroup_update_mbuf(inp, m);
2186         else
2187                 in_pcbgroup_update(inp);
2188 #endif
2189 }
2190
2191 void
2192 in_pcbrehash(struct inpcb *inp)
2193 {
2194
2195         in_pcbrehash_mbuf(inp, NULL);
2196 }
2197
2198 /*
2199  * Remove PCB from various lists.
2200  */
2201 static void
2202 in_pcbremlists(struct inpcb *inp)
2203 {
2204         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2205
2206 #ifdef INVARIANTS
2207         if (pcbinfo == &V_tcbinfo) {
2208                 INP_INFO_RLOCK_ASSERT(pcbinfo);
2209         } else {
2210                 INP_INFO_WLOCK_ASSERT(pcbinfo);
2211         }
2212 #endif
2213
2214         INP_WLOCK_ASSERT(inp);
2215         INP_LIST_WLOCK_ASSERT(pcbinfo);
2216
2217         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
2218         if (inp->inp_flags & INP_INHASHLIST) {
2219                 struct inpcbport *phd = inp->inp_phd;
2220
2221                 INP_HASH_WLOCK(pcbinfo);
2222                 LIST_REMOVE(inp, inp_hash);
2223                 LIST_REMOVE(inp, inp_portlist);
2224                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
2225                         LIST_REMOVE(phd, phd_hash);
2226                         free(phd, M_PCB);
2227                 }
2228                 INP_HASH_WUNLOCK(pcbinfo);
2229                 inp->inp_flags &= ~INP_INHASHLIST;
2230         }
2231         LIST_REMOVE(inp, inp_list);
2232         pcbinfo->ipi_count--;
2233 #ifdef PCBGROUP
2234         in_pcbgroup_remove(inp);
2235 #endif
2236 }
2237
2238 /*
2239  * Check for alternatives when higher level complains
2240  * about service problems.  For now, invalidate cached
2241  * routing information.  If the route was created dynamically
2242  * (by a redirect), time to try a default gateway again.
2243  */
2244 void
2245 in_losing(struct inpcb *inp)
2246 {
2247
2248         RO_RTFREE(&inp->inp_route);
2249         if (inp->inp_route.ro_lle)
2250                 LLE_FREE(inp->inp_route.ro_lle);        /* zeros ro_lle */
2251         return;
2252 }
2253
2254 /*
2255  * A set label operation has occurred at the socket layer, propagate the
2256  * label change into the in_pcb for the socket.
2257  */
2258 void
2259 in_pcbsosetlabel(struct socket *so)
2260 {
2261 #ifdef MAC
2262         struct inpcb *inp;
2263
2264         inp = sotoinpcb(so);
2265         KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2266
2267         INP_WLOCK(inp);
2268         SOCK_LOCK(so);
2269         mac_inpcb_sosetlabel(so, inp);
2270         SOCK_UNLOCK(so);
2271         INP_WUNLOCK(inp);
2272 #endif
2273 }
2274
2275 /*
2276  * ipport_tick runs once per second, determining if random port allocation
2277  * should be continued.  If more than ipport_randomcps ports have been
2278  * allocated in the last second, then we return to sequential port
2279  * allocation. We return to random allocation only once we drop below
2280  * ipport_randomcps for at least ipport_randomtime seconds.
2281  */
2282 static void
2283 ipport_tick(void *xtp)
2284 {
2285         VNET_ITERATOR_DECL(vnet_iter);
2286
2287         VNET_LIST_RLOCK_NOSLEEP();
2288         VNET_FOREACH(vnet_iter) {
2289                 CURVNET_SET(vnet_iter); /* XXX appease INVARIANTS here */
2290                 if (V_ipport_tcpallocs <=
2291                     V_ipport_tcplastcount + V_ipport_randomcps) {
2292                         if (V_ipport_stoprandom > 0)
2293                                 V_ipport_stoprandom--;
2294                 } else
2295                         V_ipport_stoprandom = V_ipport_randomtime;
2296                 V_ipport_tcplastcount = V_ipport_tcpallocs;
2297                 CURVNET_RESTORE();
2298         }
2299         VNET_LIST_RUNLOCK_NOSLEEP();
2300         callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
2301 }
2302
2303 static void
2304 ip_fini(void *xtp)
2305 {
2306
2307         callout_stop(&ipport_tick_callout);
2308 }
2309
2310 /*
2311  * The ipport_callout should start running at about the time we attach the
2312  * inet or inet6 domains.
2313  */
2314 static void
2315 ipport_tick_init(const void *unused __unused)
2316 {
2317
2318         /* Start ipport_tick. */
2319         callout_init(&ipport_tick_callout, 1);
2320         callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
2321         EVENTHANDLER_REGISTER(shutdown_pre_sync, ip_fini, NULL,
2322                 SHUTDOWN_PRI_DEFAULT);
2323 }
2324 SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
2325     ipport_tick_init, NULL);
2326
2327 void
2328 inp_wlock(struct inpcb *inp)
2329 {
2330
2331         INP_WLOCK(inp);
2332 }
2333
2334 void
2335 inp_wunlock(struct inpcb *inp)
2336 {
2337
2338         INP_WUNLOCK(inp);
2339 }
2340
2341 void
2342 inp_rlock(struct inpcb *inp)
2343 {
2344
2345         INP_RLOCK(inp);
2346 }
2347
2348 void
2349 inp_runlock(struct inpcb *inp)
2350 {
2351
2352         INP_RUNLOCK(inp);
2353 }
2354
2355 #ifdef INVARIANTS
2356 void
2357 inp_lock_assert(struct inpcb *inp)
2358 {
2359
2360         INP_WLOCK_ASSERT(inp);
2361 }
2362
2363 void
2364 inp_unlock_assert(struct inpcb *inp)
2365 {
2366
2367         INP_UNLOCK_ASSERT(inp);
2368 }
2369 #endif
2370
2371 void
2372 inp_apply_all(void (*func)(struct inpcb *, void *), void *arg)
2373 {
2374         struct inpcb *inp;
2375
2376         INP_INFO_WLOCK(&V_tcbinfo);
2377         LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) {
2378                 INP_WLOCK(inp);
2379                 func(inp, arg);
2380                 INP_WUNLOCK(inp);
2381         }
2382         INP_INFO_WUNLOCK(&V_tcbinfo);
2383 }
2384
2385 struct socket *
2386 inp_inpcbtosocket(struct inpcb *inp)
2387 {
2388
2389         INP_WLOCK_ASSERT(inp);
2390         return (inp->inp_socket);
2391 }
2392
2393 struct tcpcb *
2394 inp_inpcbtotcpcb(struct inpcb *inp)
2395 {
2396
2397         INP_WLOCK_ASSERT(inp);
2398         return ((struct tcpcb *)inp->inp_ppcb);
2399 }
2400
2401 int
2402 inp_ip_tos_get(const struct inpcb *inp)
2403 {
2404
2405         return (inp->inp_ip_tos);
2406 }
2407
2408 void
2409 inp_ip_tos_set(struct inpcb *inp, int val)
2410 {
2411
2412         inp->inp_ip_tos = val;
2413 }
2414
2415 void
2416 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2417     uint32_t *faddr, uint16_t *fp)
2418 {
2419
2420         INP_LOCK_ASSERT(inp);
2421         *laddr = inp->inp_laddr.s_addr;
2422         *faddr = inp->inp_faddr.s_addr;
2423         *lp = inp->inp_lport;
2424         *fp = inp->inp_fport;
2425 }
2426
2427 struct inpcb *
2428 so_sotoinpcb(struct socket *so)
2429 {
2430
2431         return (sotoinpcb(so));
2432 }
2433
2434 struct tcpcb *
2435 so_sototcpcb(struct socket *so)
2436 {
2437
2438         return (sototcpcb(so));
2439 }
2440
2441 #ifdef DDB
2442 static void
2443 db_print_indent(int indent)
2444 {
2445         int i;
2446
2447         for (i = 0; i < indent; i++)
2448                 db_printf(" ");
2449 }
2450
2451 static void
2452 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
2453 {
2454         char faddr_str[48], laddr_str[48];
2455
2456         db_print_indent(indent);
2457         db_printf("%s at %p\n", name, inc);
2458
2459         indent += 2;
2460
2461 #ifdef INET6
2462         if (inc->inc_flags & INC_ISIPV6) {
2463                 /* IPv6. */
2464                 ip6_sprintf(laddr_str, &inc->inc6_laddr);
2465                 ip6_sprintf(faddr_str, &inc->inc6_faddr);
2466         } else
2467 #endif
2468         {
2469                 /* IPv4. */
2470                 inet_ntoa_r(inc->inc_laddr, laddr_str);
2471                 inet_ntoa_r(inc->inc_faddr, faddr_str);
2472         }
2473         db_print_indent(indent);
2474         db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
2475             ntohs(inc->inc_lport));
2476         db_print_indent(indent);
2477         db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
2478             ntohs(inc->inc_fport));
2479 }
2480
2481 static void
2482 db_print_inpflags(int inp_flags)
2483 {
2484         int comma;
2485
2486         comma = 0;
2487         if (inp_flags & INP_RECVOPTS) {
2488                 db_printf("%sINP_RECVOPTS", comma ? ", " : "");
2489                 comma = 1;
2490         }
2491         if (inp_flags & INP_RECVRETOPTS) {
2492                 db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
2493                 comma = 1;
2494         }
2495         if (inp_flags & INP_RECVDSTADDR) {
2496                 db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
2497                 comma = 1;
2498         }
2499         if (inp_flags & INP_HDRINCL) {
2500                 db_printf("%sINP_HDRINCL", comma ? ", " : "");
2501                 comma = 1;
2502         }
2503         if (inp_flags & INP_HIGHPORT) {
2504                 db_printf("%sINP_HIGHPORT", comma ? ", " : "");
2505                 comma = 1;
2506         }
2507         if (inp_flags & INP_LOWPORT) {
2508                 db_printf("%sINP_LOWPORT", comma ? ", " : "");
2509                 comma = 1;
2510         }
2511         if (inp_flags & INP_ANONPORT) {
2512                 db_printf("%sINP_ANONPORT", comma ? ", " : "");
2513                 comma = 1;
2514         }
2515         if (inp_flags & INP_RECVIF) {
2516                 db_printf("%sINP_RECVIF", comma ? ", " : "");
2517                 comma = 1;
2518         }
2519         if (inp_flags & INP_MTUDISC) {
2520                 db_printf("%sINP_MTUDISC", comma ? ", " : "");
2521                 comma = 1;
2522         }
2523         if (inp_flags & INP_RECVTTL) {
2524                 db_printf("%sINP_RECVTTL", comma ? ", " : "");
2525                 comma = 1;
2526         }
2527         if (inp_flags & INP_DONTFRAG) {
2528                 db_printf("%sINP_DONTFRAG", comma ? ", " : "");
2529                 comma = 1;
2530         }
2531         if (inp_flags & INP_RECVTOS) {
2532                 db_printf("%sINP_RECVTOS", comma ? ", " : "");
2533                 comma = 1;
2534         }
2535         if (inp_flags & IN6P_IPV6_V6ONLY) {
2536                 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
2537                 comma = 1;
2538         }
2539         if (inp_flags & IN6P_PKTINFO) {
2540                 db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
2541                 comma = 1;
2542         }
2543         if (inp_flags & IN6P_HOPLIMIT) {
2544                 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
2545                 comma = 1;
2546         }
2547         if (inp_flags & IN6P_HOPOPTS) {
2548                 db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
2549                 comma = 1;
2550         }
2551         if (inp_flags & IN6P_DSTOPTS) {
2552                 db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
2553                 comma = 1;
2554         }
2555         if (inp_flags & IN6P_RTHDR) {
2556                 db_printf("%sIN6P_RTHDR", comma ? ", " : "");
2557                 comma = 1;
2558         }
2559         if (inp_flags & IN6P_RTHDRDSTOPTS) {
2560                 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
2561                 comma = 1;
2562         }
2563         if (inp_flags & IN6P_TCLASS) {
2564                 db_printf("%sIN6P_TCLASS", comma ? ", " : "");
2565                 comma = 1;
2566         }
2567         if (inp_flags & IN6P_AUTOFLOWLABEL) {
2568                 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
2569                 comma = 1;
2570         }
2571         if (inp_flags & INP_TIMEWAIT) {
2572                 db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
2573                 comma  = 1;
2574         }
2575         if (inp_flags & INP_ONESBCAST) {
2576                 db_printf("%sINP_ONESBCAST", comma ? ", " : "");
2577                 comma  = 1;
2578         }
2579         if (inp_flags & INP_DROPPED) {
2580                 db_printf("%sINP_DROPPED", comma ? ", " : "");
2581                 comma  = 1;
2582         }
2583         if (inp_flags & INP_SOCKREF) {
2584                 db_printf("%sINP_SOCKREF", comma ? ", " : "");
2585                 comma  = 1;
2586         }
2587         if (inp_flags & IN6P_RFC2292) {
2588                 db_printf("%sIN6P_RFC2292", comma ? ", " : "");
2589                 comma = 1;
2590         }
2591         if (inp_flags & IN6P_MTU) {
2592                 db_printf("IN6P_MTU%s", comma ? ", " : "");
2593                 comma = 1;
2594         }
2595 }
2596
2597 static void
2598 db_print_inpvflag(u_char inp_vflag)
2599 {
2600         int comma;
2601
2602         comma = 0;
2603         if (inp_vflag & INP_IPV4) {
2604                 db_printf("%sINP_IPV4", comma ? ", " : "");
2605                 comma  = 1;
2606         }
2607         if (inp_vflag & INP_IPV6) {
2608                 db_printf("%sINP_IPV6", comma ? ", " : "");
2609                 comma  = 1;
2610         }
2611         if (inp_vflag & INP_IPV6PROTO) {
2612                 db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
2613                 comma  = 1;
2614         }
2615 }
2616
2617 static void
2618 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
2619 {
2620
2621         db_print_indent(indent);
2622         db_printf("%s at %p\n", name, inp);
2623
2624         indent += 2;
2625
2626         db_print_indent(indent);
2627         db_printf("inp_flow: 0x%x\n", inp->inp_flow);
2628
2629         db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
2630
2631         db_print_indent(indent);
2632         db_printf("inp_ppcb: %p   inp_pcbinfo: %p   inp_socket: %p\n",
2633             inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
2634
2635         db_print_indent(indent);
2636         db_printf("inp_label: %p   inp_flags: 0x%x (",
2637            inp->inp_label, inp->inp_flags);
2638         db_print_inpflags(inp->inp_flags);
2639         db_printf(")\n");
2640
2641         db_print_indent(indent);
2642         db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
2643             inp->inp_vflag);
2644         db_print_inpvflag(inp->inp_vflag);
2645         db_printf(")\n");
2646
2647         db_print_indent(indent);
2648         db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
2649             inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
2650
2651         db_print_indent(indent);
2652 #ifdef INET6
2653         if (inp->inp_vflag & INP_IPV6) {
2654                 db_printf("in6p_options: %p   in6p_outputopts: %p   "
2655                     "in6p_moptions: %p\n", inp->in6p_options,
2656                     inp->in6p_outputopts, inp->in6p_moptions);
2657                 db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
2658                     "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
2659                     inp->in6p_hops);
2660         } else
2661 #endif
2662         {
2663                 db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
2664                     "inp_ip_moptions: %p\n", inp->inp_ip_tos,
2665                     inp->inp_options, inp->inp_moptions);
2666         }
2667
2668         db_print_indent(indent);
2669         db_printf("inp_phd: %p   inp_gencnt: %ju\n", inp->inp_phd,
2670             (uintmax_t)inp->inp_gencnt);
2671 }
2672
2673 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
2674 {
2675         struct inpcb *inp;
2676
2677         if (!have_addr) {
2678                 db_printf("usage: show inpcb <addr>\n");
2679                 return;
2680         }
2681         inp = (struct inpcb *)addr;
2682
2683         db_print_inpcb(inp, "inpcb", 0);
2684 }
2685 #endif /* DDB */
2686
2687 #ifdef RATELIMIT
2688 /*
2689  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
2690  * if any.
2691  */
2692 int
2693 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
2694 {
2695         union if_snd_tag_modify_params params = {
2696                 .rate_limit.max_rate = max_pacing_rate,
2697         };
2698         struct m_snd_tag *mst;
2699         struct ifnet *ifp;
2700         int error;
2701
2702         mst = inp->inp_snd_tag;
2703         if (mst == NULL)
2704                 return (EINVAL);
2705
2706         ifp = mst->ifp;
2707         if (ifp == NULL)
2708                 return (EINVAL);
2709
2710         if (ifp->if_snd_tag_modify == NULL) {
2711                 error = EOPNOTSUPP;
2712         } else {
2713                 error = ifp->if_snd_tag_modify(mst, &params);
2714         }
2715         return (error);
2716 }
2717
2718 /*
2719  * Query existing TX rate limit based on the existing
2720  * "inp->inp_snd_tag", if any.
2721  */
2722 int
2723 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
2724 {
2725         union if_snd_tag_query_params params = { };
2726         struct m_snd_tag *mst;
2727         struct ifnet *ifp;
2728         int error;
2729
2730         mst = inp->inp_snd_tag;
2731         if (mst == NULL)
2732                 return (EINVAL);
2733
2734         ifp = mst->ifp;
2735         if (ifp == NULL)
2736                 return (EINVAL);
2737
2738         if (ifp->if_snd_tag_query == NULL) {
2739                 error = EOPNOTSUPP;
2740         } else {
2741                 error = ifp->if_snd_tag_query(mst, &params);
2742                 if (error == 0 &&  p_max_pacing_rate != NULL)
2743                         *p_max_pacing_rate = params.rate_limit.max_rate;
2744         }
2745         return (error);
2746 }
2747
2748 /*
2749  * Allocate a new TX rate limit send tag from the network interface
2750  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
2751  */
2752 int
2753 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
2754     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate)
2755 {
2756         union if_snd_tag_alloc_params params = {
2757                 .rate_limit.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT,
2758                 .rate_limit.hdr.flowid = flowid,
2759                 .rate_limit.hdr.flowtype = flowtype,
2760                 .rate_limit.max_rate = max_pacing_rate,
2761         };
2762         int error;
2763
2764         INP_WLOCK_ASSERT(inp);
2765
2766         if (inp->inp_snd_tag != NULL)
2767                 return (EINVAL);
2768
2769         if (ifp->if_snd_tag_alloc == NULL) {
2770                 error = EOPNOTSUPP;
2771         } else {
2772                 error = ifp->if_snd_tag_alloc(ifp, &params, &inp->inp_snd_tag);
2773
2774                 /*
2775                  * At success increment the refcount on
2776                  * the send tag's network interface:
2777                  */
2778                 if (error == 0)
2779                         if_ref(inp->inp_snd_tag->ifp);
2780         }
2781         return (error);
2782 }
2783
2784 /*
2785  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
2786  * if any:
2787  */
2788 void
2789 in_pcbdetach_txrtlmt(struct inpcb *inp)
2790 {
2791         struct m_snd_tag *mst;
2792         struct ifnet *ifp;
2793
2794         INP_WLOCK_ASSERT(inp);
2795
2796         mst = inp->inp_snd_tag;
2797         inp->inp_snd_tag = NULL;
2798
2799         if (mst == NULL)
2800                 return;
2801
2802         ifp = mst->ifp;
2803         if (ifp == NULL)
2804                 return;
2805
2806         /*
2807          * If the device was detached while we still had reference(s)
2808          * on the ifp, we assume if_snd_tag_free() was replaced with
2809          * stubs.
2810          */
2811         ifp->if_snd_tag_free(mst);
2812
2813         /* release reference count on network interface */
2814         if_rele(ifp);
2815 }
2816
2817 /*
2818  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
2819  * is set in the fast path and will attach/detach/modify the TX rate
2820  * limit send tag based on the socket's so_max_pacing_rate value.
2821  */
2822 void
2823 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
2824 {
2825         struct socket *socket;
2826         uint32_t max_pacing_rate;
2827         bool did_upgrade;
2828         int error;
2829
2830         if (inp == NULL)
2831                 return;
2832
2833         socket = inp->inp_socket;
2834         if (socket == NULL)
2835                 return;
2836
2837         if (!INP_WLOCKED(inp)) {
2838                 /*
2839                  * NOTE: If the write locking fails, we need to bail
2840                  * out and use the non-ratelimited ring for the
2841                  * transmit until there is a new chance to get the
2842                  * write lock.
2843                  */
2844                 if (!INP_TRY_UPGRADE(inp))
2845                         return;
2846                 did_upgrade = 1;
2847         } else {
2848                 did_upgrade = 0;
2849         }
2850
2851         /*
2852          * NOTE: The so_max_pacing_rate value is read unlocked,
2853          * because atomic updates are not required since the variable
2854          * is checked at every mbuf we send. It is assumed that the
2855          * variable read itself will be atomic.
2856          */
2857         max_pacing_rate = socket->so_max_pacing_rate;
2858
2859         /*
2860          * NOTE: When attaching to a network interface a reference is
2861          * made to ensure the network interface doesn't go away until
2862          * all ratelimit connections are gone. The network interface
2863          * pointers compared below represent valid network interfaces,
2864          * except when comparing towards NULL.
2865          */
2866         if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
2867                 error = 0;
2868         } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
2869                 if (inp->inp_snd_tag != NULL)
2870                         in_pcbdetach_txrtlmt(inp);
2871                 error = 0;
2872         } else if (inp->inp_snd_tag == NULL) {
2873                 /*
2874                  * In order to utilize packet pacing with RSS, we need
2875                  * to wait until there is a valid RSS hash before we
2876                  * can proceed:
2877                  */
2878                 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
2879                         error = EAGAIN;
2880                 } else {
2881                         error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
2882                             mb->m_pkthdr.flowid, max_pacing_rate);
2883                 }
2884         } else {
2885                 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
2886         }
2887         if (error == 0 || error == EOPNOTSUPP)
2888                 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
2889         if (did_upgrade)
2890                 INP_DOWNGRADE(inp);
2891 }
2892
2893 /*
2894  * Track route changes for TX rate limiting.
2895  */
2896 void
2897 in_pcboutput_eagain(struct inpcb *inp)
2898 {
2899         struct socket *socket;
2900         bool did_upgrade;
2901
2902         if (inp == NULL)
2903                 return;
2904
2905         socket = inp->inp_socket;
2906         if (socket == NULL)
2907                 return;
2908
2909         if (inp->inp_snd_tag == NULL)
2910                 return;
2911
2912         if (!INP_WLOCKED(inp)) {
2913                 /*
2914                  * NOTE: If the write locking fails, we need to bail
2915                  * out and use the non-ratelimited ring for the
2916                  * transmit until there is a new chance to get the
2917                  * write lock.
2918                  */
2919                 if (!INP_TRY_UPGRADE(inp))
2920                         return;
2921                 did_upgrade = 1;
2922         } else {
2923                 did_upgrade = 0;
2924         }
2925
2926         /* detach rate limiting */
2927         in_pcbdetach_txrtlmt(inp);
2928
2929         /* make sure new mbuf send tag allocation is made */
2930         inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
2931
2932         if (did_upgrade)
2933                 INP_DOWNGRADE(inp);
2934 }
2935 #endif /* RATELIMIT */