sys/netinet/in_pcb.c

   1 /*-
   2  * Copyright (c) 1982, 1986, 1991, 1993, 1995
   3  *      The Regents of the University of California.
   4  * Copyright (c) 2007-2009 Robert N. M. Watson
   5  * Copyright (c) 2010-2011 Juniper Networks, Inc.
   6  * All rights reserved.
   7  *
   8  * Portions of this software were developed by Robert N. M. Watson under
   9  * contract to Juniper Networks, Inc.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 3. Neither the name of the University nor the names of its contributors
  20  *    may be used to endorse or promote products derived from this software
  21  *    without specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  26  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  27  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  31  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  32  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  *
  35  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  36  */
  37
  38 #include <sys/cdefs.h>
  39 __FBSDID("$FreeBSD$");
  40
  41 #include "opt_ddb.h"
  42 #include "opt_ipsec.h"
  43 #include "opt_inet.h"
  44 #include "opt_inet6.h"
  45 #include "opt_ratelimit.h"
  46 #include "opt_pcbgroup.h"
  47 #include "opt_rss.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/systm.h>
  51 #include <sys/lock.h>
  52 #include <sys/malloc.h>
  53 #include <sys/mbuf.h>
  54 #include <sys/callout.h>
  55 #include <sys/eventhandler.h>
  56 #include <sys/domain.h>
  57 #include <sys/protosw.h>
  58 #include <sys/rmlock.h>
  59 #include <sys/socket.h>
  60 #include <sys/socketvar.h>
  61 #include <sys/sockio.h>
  62 #include <sys/priv.h>
  63 #include <sys/proc.h>
  64 #include <sys/refcount.h>
  65 #include <sys/jail.h>
  66 #include <sys/kernel.h>
  67 #include <sys/sysctl.h>
  68
  69 #ifdef DDB
  70 #include <ddb/ddb.h>
  71 #endif
  72
  73 #include <vm/uma.h>
  74
  75 #include <net/if.h>
  76 #include <net/if_var.h>
  77 #include <net/if_types.h>
  78 #include <net/if_llatbl.h>
  79 #include <net/route.h>
  80 #include <net/rss_config.h>
  81 #include <net/vnet.h>
  82
  83 #if defined(INET) || defined(INET6)
  84 #include <netinet/in.h>
  85 #include <netinet/in_pcb.h>
  86 #include <netinet/ip_var.h>
  87 #include <netinet/tcp_var.h>
  88 #include <netinet/udp.h>
  89 #include <netinet/udp_var.h>
  90 #endif
  91 #ifdef INET
  92 #include <netinet/in_var.h>
  93 #endif
  94 #ifdef INET6
  95 #include <netinet/ip6.h>
  96 #include <netinet6/in6_pcb.h>
  97 #include <netinet6/in6_var.h>
  98 #include <netinet6/ip6_var.h>
  99 #endif /* INET6 */
 100
 101 #include <netipsec/ipsec_support.h>
 102
 103 #include <security/mac/mac_framework.h>
 104
 105 static struct callout   ipport_tick_callout;
 106
 107 /*
 108  * These configure the range of local port addresses assigned to
 109  * "unspecified" outgoing connections/packets/whatever.
 110  */
 111 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;    /* 1023 */
 112 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;    /* 600 */
 113 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;     /* 10000 */
 114 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;       /* 65535 */
 115 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;      /* 49152 */
 116 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;        /* 65535 */
 117
 118 /*
 119  * Reserved ports accessible only to root. There are significant
 120  * security considerations that must be accounted for when changing these,
 121  * but the security benefits can be great. Please be careful.
 122  */
 123 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;    /* 1023 */
 124 VNET_DEFINE(int, ipport_reservedlow);
 125
 126 /* Variables dealing with random ephemeral port allocation. */
 127 VNET_DEFINE(int, ipport_randomized) = 1;        /* user controlled via sysctl */
 128 VNET_DEFINE(int, ipport_randomcps) = 10;        /* user controlled via sysctl */
 129 VNET_DEFINE(int, ipport_randomtime) = 45;       /* user controlled via sysctl */
 130 VNET_DEFINE(int, ipport_stoprandom);            /* toggled by ipport_tick */
 131 VNET_DEFINE(int, ipport_tcpallocs);
 132 static VNET_DEFINE(int, ipport_tcplastcount);
 133
 134 #define V_ipport_tcplastcount           VNET(ipport_tcplastcount)
 135
 136 static void     in_pcbremlists(struct inpcb *inp);
 137 #ifdef INET
 138 static struct inpcb     *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
 139                             struct in_addr faddr, u_int fport_arg,
 140                             struct in_addr laddr, u_int lport_arg,
 141                             int lookupflags, struct ifnet *ifp);
 142
 143 #define RANGECHK(var, min, max) \
 144         if ((var) < (min)) { (var) = (min); } \
 145         else if ((var) > (max)) { (var) = (max); }
 146
 147 static int
 148 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 149 {
 150         int error;
 151
 152         error = sysctl_handle_int(oidp, arg1, arg2, req);
 153         if (error == 0) {
 154                 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 155                 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 156                 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
 157                 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
 158                 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
 159                 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
 160         }
 161         return (error);
 162 }
 163
 164 #undef RANGECHK
 165
 166 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange, CTLFLAG_RW, 0,
 167     "IP Ports");
 168
 169 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
 170         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 171         &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I", "");
 172 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
 173         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 174         &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I", "");
 175 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
 176         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 177         &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I", "");
 178 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
 179         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 180         &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I", "");
 181 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
 182         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 183         &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I", "");
 184 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
 185         CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW,
 186         &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I", "");
 187 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
 188         CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
 189         &VNET_NAME(ipport_reservedhigh), 0, "");
 190 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
 191         CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
 192 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
 193         CTLFLAG_VNET | CTLFLAG_RW,
 194         &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
 195 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
 196         CTLFLAG_VNET | CTLFLAG_RW,
 197         &VNET_NAME(ipport_randomcps), 0, "Maximum number of random port "
 198         "allocations before switching to a sequental one");
 199 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
 200         CTLFLAG_VNET | CTLFLAG_RW,
 201         &VNET_NAME(ipport_randomtime), 0,
 202         "Minimum time to keep sequental port "
 203         "allocation before switching to a random one");
 204 #endif /* INET */
 205
 206 /*
 207  * in_pcb.c: manage the Protocol Control Blocks.
 208  *
 209  * NOTE: It is assumed that most of these functions will be called with
 210  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
 211  * functions often modify hash chains or addresses in pcbs.
 212  */
 213
 214 /*
 215  * Initialize an inpcbinfo -- we should be able to reduce the number of
 216  * arguments in time.
 217  */
 218 void
 219 in_pcbinfo_init(struct inpcbinfo *pcbinfo, const char *name,
 220     struct inpcbhead *listhead, int hash_nelements, int porthash_nelements,
 221     char *inpcbzone_name, uma_init inpcbzone_init, uma_fini inpcbzone_fini,
 222     uint32_t inpcbzone_flags, u_int hashfields)
 223 {
 224
 225         INP_INFO_LOCK_INIT(pcbinfo, name);
 226         INP_HASH_LOCK_INIT(pcbinfo, "pcbinfohash");     /* XXXRW: argument? */
 227         INP_LIST_LOCK_INIT(pcbinfo, "pcbinfolist");
 228 #ifdef VIMAGE
 229         pcbinfo->ipi_vnet = curvnet;
 230 #endif
 231         pcbinfo->ipi_listhead = listhead;
 232         LIST_INIT(pcbinfo->ipi_listhead);
 233         pcbinfo->ipi_count = 0;
 234         pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB,
 235             &pcbinfo->ipi_hashmask);
 236         pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
 237             &pcbinfo->ipi_porthashmask);
 238 #ifdef PCBGROUP
 239         in_pcbgroup_init(pcbinfo, hashfields, hash_nelements);
 240 #endif
 241         pcbinfo->ipi_zone = uma_zcreate(inpcbzone_name, sizeof(struct inpcb),
 242             NULL, NULL, inpcbzone_init, inpcbzone_fini, UMA_ALIGN_PTR,
 243             inpcbzone_flags);
 244         uma_zone_set_max(pcbinfo->ipi_zone, maxsockets);
 245         uma_zone_set_warning(pcbinfo->ipi_zone,
 246             "kern.ipc.maxsockets limit reached");
 247 }
 248
 249 /*
 250  * Destroy an inpcbinfo.
 251  */
 252 void
 253 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
 254 {
 255
 256         KASSERT(pcbinfo->ipi_count == 0,
 257             ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
 258
 259         hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask);
 260         hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
 261             pcbinfo->ipi_porthashmask);
 262 #ifdef PCBGROUP
 263         in_pcbgroup_destroy(pcbinfo);
 264 #endif
 265         uma_zdestroy(pcbinfo->ipi_zone);
 266         INP_LIST_LOCK_DESTROY(pcbinfo);
 267         INP_HASH_LOCK_DESTROY(pcbinfo);
 268         INP_INFO_LOCK_DESTROY(pcbinfo);
 269 }
 270
 271 /*
 272  * Allocate a PCB and associate it with the socket.
 273  * On success return with the PCB locked.
 274  */
 275 int
 276 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 277 {
 278         struct inpcb *inp;
 279         int error;
 280
 281 #ifdef INVARIANTS
 282         if (pcbinfo == &V_tcbinfo) {
 283                 INP_INFO_RLOCK_ASSERT(pcbinfo);
 284         } else {
 285                 INP_INFO_WLOCK_ASSERT(pcbinfo);
 286         }
 287 #endif
 288
 289         error = 0;
 290         inp = uma_zalloc(pcbinfo->ipi_zone, M_NOWAIT);
 291         if (inp == NULL)
 292                 return (ENOBUFS);
 293         bzero(inp, inp_zero_size);
 294         inp->inp_pcbinfo = pcbinfo;
 295         inp->inp_socket = so;
 296         inp->inp_cred = crhold(so->so_cred);
 297         inp->inp_inc.inc_fibnum = so->so_fibnum;
 298 #ifdef MAC
 299         error = mac_inpcb_init(inp, M_NOWAIT);
 300         if (error != 0)
 301                 goto out;
 302         mac_inpcb_create(so, inp);
 303 #endif
 304 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
 305         error = ipsec_init_pcbpolicy(inp);
 306         if (error != 0) {
 307 #ifdef MAC
 308                 mac_inpcb_destroy(inp);
 309 #endif
 310                 goto out;
 311         }
 312 #endif /*IPSEC*/
 313 #ifdef INET6
 314         if (INP_SOCKAF(so) == AF_INET6) {
 315                 inp->inp_vflag |= INP_IPV6PROTO;
 316                 if (V_ip6_v6only)
 317                         inp->inp_flags |= IN6P_IPV6_V6ONLY;
 318         }
 319 #endif
 320         INP_WLOCK(inp);
 321         INP_LIST_WLOCK(pcbinfo);
 322         LIST_INSERT_HEAD(pcbinfo->ipi_listhead, inp, inp_list);
 323         pcbinfo->ipi_count++;
 324         so->so_pcb = (caddr_t)inp;
 325 #ifdef INET6
 326         if (V_ip6_auto_flowlabel)
 327                 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 328 #endif
 329         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 330         refcount_init(&inp->inp_refcount, 1);   /* Reference from inpcbinfo */
 331         INP_LIST_WUNLOCK(pcbinfo);
 332 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
 333 out:
 334         if (error != 0) {
 335                 crfree(inp->inp_cred);
 336                 uma_zfree(pcbinfo->ipi_zone, inp);
 337         }
 338 #endif
 339         return (error);
 340 }
 341
 342 #ifdef INET
 343 int
 344 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 345 {
 346         int anonport, error;
 347
 348         INP_WLOCK_ASSERT(inp);
 349         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 350
 351         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 352                 return (EINVAL);
 353         anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
 354         error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
 355             &inp->inp_lport, cred);
 356         if (error)
 357                 return (error);
 358         if (in_pcbinshash(inp) != 0) {
 359                 inp->inp_laddr.s_addr = INADDR_ANY;
 360                 inp->inp_lport = 0;
 361                 return (EAGAIN);
 362         }
 363         if (anonport)
 364                 inp->inp_flags |= INP_ANONPORT;
 365         return (0);
 366 }
 367 #endif
 368
 369 /*
 370  * Select a local port (number) to use.
 371  */
 372 #if defined(INET) || defined(INET6)
 373 int
 374 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
 375     struct ucred *cred, int lookupflags)
 376 {
 377         struct inpcbinfo *pcbinfo;
 378         struct inpcb *tmpinp;
 379         unsigned short *lastport;
 380         int count, dorandom, error;
 381         u_short aux, first, last, lport;
 382 #ifdef INET
 383         struct in_addr laddr;
 384 #endif
 385
 386         pcbinfo = inp->inp_pcbinfo;
 387
 388         /*
 389          * Because no actual state changes occur here, a global write lock on
 390          * the pcbinfo isn't required.
 391          */
 392         INP_LOCK_ASSERT(inp);
 393         INP_HASH_LOCK_ASSERT(pcbinfo);
 394
 395         if (inp->inp_flags & INP_HIGHPORT) {
 396                 first = V_ipport_hifirstauto;   /* sysctl */
 397                 last  = V_ipport_hilastauto;
 398                 lastport = &pcbinfo->ipi_lasthi;
 399         } else if (inp->inp_flags & INP_LOWPORT) {
 400                 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT, 0);
 401                 if (error)
 402                         return (error);
 403                 first = V_ipport_lowfirstauto;  /* 1023 */
 404                 last  = V_ipport_lowlastauto;   /* 600 */
 405                 lastport = &pcbinfo->ipi_lastlow;
 406         } else {
 407                 first = V_ipport_firstauto;     /* sysctl */
 408                 last  = V_ipport_lastauto;
 409                 lastport = &pcbinfo->ipi_lastport;
 410         }
 411         /*
 412          * For UDP(-Lite), use random port allocation as long as the user
 413          * allows it.  For TCP (and as of yet unknown) connections,
 414          * use random port allocation only if the user allows it AND
 415          * ipport_tick() allows it.
 416          */
 417         if (V_ipport_randomized &&
 418                 (!V_ipport_stoprandom || pcbinfo == &V_udbinfo ||
 419                 pcbinfo == &V_ulitecbinfo))
 420                 dorandom = 1;
 421         else
 422                 dorandom = 0;
 423         /*
 424          * It makes no sense to do random port allocation if
 425          * we have the only port available.
 426          */
 427         if (first == last)
 428                 dorandom = 0;
 429         /* Make sure to not include UDP(-Lite) packets in the count. */
 430         if (pcbinfo != &V_udbinfo || pcbinfo != &V_ulitecbinfo)
 431                 V_ipport_tcpallocs++;
 432         /*
 433          * Instead of having two loops further down counting up or down
 434          * make sure that first is always <= last and go with only one
 435          * code path implementing all logic.
 436          */
 437         if (first > last) {
 438                 aux = first;
 439                 first = last;
 440                 last = aux;
 441         }
 442
 443 #ifdef INET
 444         /* Make the compiler happy. */
 445         laddr.s_addr = 0;
 446         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
 447                 KASSERT(laddrp != NULL, ("%s: laddrp NULL for v4 inp %p",
 448                     __func__, inp));
 449                 laddr = *laddrp;
 450         }
 451 #endif
 452         tmpinp = NULL;  /* Make compiler happy. */
 453         lport = *lportp;
 454
 455         if (dorandom)
 456                 *lastport = first + (arc4random() % (last - first));
 457
 458         count = last - first;
 459
 460         do {
 461                 if (count-- < 0)        /* completely used? */
 462                         return (EADDRNOTAVAIL);
 463                 ++*lastport;
 464                 if (*lastport < first || *lastport > last)
 465                         *lastport = first;
 466                 lport = htons(*lastport);
 467
 468 #ifdef INET6
 469                 if ((inp->inp_vflag & INP_IPV6) != 0)
 470                         tmpinp = in6_pcblookup_local(pcbinfo,
 471                             &inp->in6p_laddr, lport, lookupflags, cred);
 472 #endif
 473 #if defined(INET) && defined(INET6)
 474                 else
 475 #endif
 476 #ifdef INET
 477                         tmpinp = in_pcblookup_local(pcbinfo, laddr,
 478                             lport, lookupflags, cred);
 479 #endif
 480         } while (tmpinp != NULL);
 481
 482 #ifdef INET
 483         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4)
 484                 laddrp->s_addr = laddr.s_addr;
 485 #endif
 486         *lportp = lport;
 487
 488         return (0);
 489 }
 490
 491 /*
 492  * Return cached socket options.
 493  */
 494 short
 495 inp_so_options(const struct inpcb *inp)
 496 {
 497    short so_options;
 498
 499    so_options = 0;
 500
 501    if ((inp->inp_flags2 & INP_REUSEPORT) != 0)
 502            so_options |= SO_REUSEPORT;
 503    if ((inp->inp_flags2 & INP_REUSEADDR) != 0)
 504            so_options |= SO_REUSEADDR;
 505    return (so_options);
 506 }
 507 #endif /* INET || INET6 */
 508
 509 /*
 510  * Check if a new BINDMULTI socket is allowed to be created.
 511  *
 512  * ni points to the new inp.
 513  * oi points to the exisitng inp.
 514  *
 515  * This checks whether the existing inp also has BINDMULTI and
 516  * whether the credentials match.
 517  */
 518 int
 519 in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
 520 {
 521         /* Check permissions match */
 522         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 523             (ni->inp_cred->cr_uid !=
 524             oi->inp_cred->cr_uid))
 525                 return (0);
 526
 527         /* Check the existing inp has BINDMULTI set */
 528         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 529             ((oi->inp_flags2 & INP_BINDMULTI) == 0))
 530                 return (0);
 531
 532         /*
 533          * We're okay - either INP_BINDMULTI isn't set on ni, or
 534          * it is and it matches the checks.
 535          */
 536         return (1);
 537 }
 538
 539 #ifdef INET
 540 /*
 541  * Set up a bind operation on a PCB, performing port allocation
 542  * as required, but do not actually modify the PCB. Callers can
 543  * either complete the bind by setting inp_laddr/inp_lport and
 544  * calling in_pcbinshash(), or they can just use the resulting
 545  * port and address to authorise the sending of a once-off packet.
 546  *
 547  * On error, the values of *laddrp and *lportp are not changed.
 548  */
 549 int
 550 in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
 551     u_short *lportp, struct ucred *cred)
 552 {
 553         struct socket *so = inp->inp_socket;
 554         struct sockaddr_in *sin;
 555         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 556         struct in_addr laddr;
 557         u_short lport = 0;
 558         int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
 559         int error;
 560
 561         /*
 562          * No state changes, so read locks are sufficient here.
 563          */
 564         INP_LOCK_ASSERT(inp);
 565         INP_HASH_LOCK_ASSERT(pcbinfo);
 566
 567         if (TAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */
 568                 return (EADDRNOTAVAIL);
 569         laddr.s_addr = *laddrp;
 570         if (nam != NULL && laddr.s_addr != INADDR_ANY)
 571                 return (EINVAL);
 572         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) == 0)
 573                 lookupflags = INPLOOKUP_WILDCARD;
 574         if (nam == NULL) {
 575                 if ((error = prison_local_ip4(cred, &laddr)) != 0)
 576                         return (error);
 577         } else {
 578                 sin = (struct sockaddr_in *)nam;
 579                 if (nam->sa_len != sizeof (*sin))
 580                         return (EINVAL);
 581 #ifdef notdef
 582                 /*
 583                  * We should check the family, but old programs
 584                  * incorrectly fail to initialize it.
 585                  */
 586                 if (sin->sin_family != AF_INET)
 587                         return (EAFNOSUPPORT);
 588 #endif
 589                 error = prison_local_ip4(cred, &sin->sin_addr);
 590                 if (error)
 591                         return (error);
 592                 if (sin->sin_port != *lportp) {
 593                         /* Don't allow the port to change. */
 594                         if (*lportp != 0)
 595                                 return (EINVAL);
 596                         lport = sin->sin_port;
 597                 }
 598                 /* NB: lport is left as 0 if the port isn't being changed. */
 599                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 600                         /*
 601                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 602                          * allow complete duplication of binding if
 603                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 604                          * and a multicast address is bound on both
 605                          * new and duplicated sockets.
 606                          */
 607                         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
 608                                 reuseport = SO_REUSEADDR|SO_REUSEPORT;
 609                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 610                         sin->sin_port = 0;              /* yech... */
 611                         bzero(&sin->sin_zero, sizeof(sin->sin_zero));
 612                         /*
 613                          * Is the address a local IP address?
 614                          * If INP_BINDANY is set, then the socket may be bound
 615                          * to any endpoint address, local or not.
 616                          */
 617                         if ((inp->inp_flags & INP_BINDANY) == 0 &&
 618                             ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
 619                                 return (EADDRNOTAVAIL);
 620                 }
 621                 laddr = sin->sin_addr;
 622                 if (lport) {
 623                         struct inpcb *t;
 624                         struct tcptw *tw;
 625
 626                         /* GROSS */
 627                         if (ntohs(lport) <= V_ipport_reservedhigh &&
 628                             ntohs(lport) >= V_ipport_reservedlow &&
 629                             priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT,
 630                             0))
 631                                 return (EACCES);
 632                         if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 633                             priv_check_cred(inp->inp_cred,
 634                             PRIV_NETINET_REUSEPORT, 0) != 0) {
 635                                 t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 636                                     lport, INPLOOKUP_WILDCARD, cred);
 637         /*
 638          * XXX
 639          * This entire block sorely needs a rewrite.
 640          */
 641                                 if (t &&
 642                                     ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 643                                     ((t->inp_flags & INP_TIMEWAIT) == 0) &&
 644                                     (so->so_type != SOCK_STREAM ||
 645                                      ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
 646                                     (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
 647                                      ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
 648                                      (t->inp_flags2 & INP_REUSEPORT) == 0) &&
 649                                     (inp->inp_cred->cr_uid !=
 650                                      t->inp_cred->cr_uid))
 651                                         return (EADDRINUSE);
 652
 653                                 /*
 654                                  * If the socket is a BINDMULTI socket, then
 655                                  * the credentials need to match and the
 656                                  * original socket also has to have been bound
 657                                  * with BINDMULTI.
 658                                  */
 659                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 660                                         return (EADDRINUSE);
 661                         }
 662                         t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 663                             lport, lookupflags, cred);
 664                         if (t && (t->inp_flags & INP_TIMEWAIT)) {
 665                                 /*
 666                                  * XXXRW: If an incpb has had its timewait
 667                                  * state recycled, we treat the address as
 668                                  * being in use (for now).  This is better
 669                                  * than a panic, but not desirable.
 670                                  */
 671                                 tw = intotw(t);
 672                                 if (tw == NULL ||
 673                                     (reuseport & tw->tw_so_options) == 0)
 674                                         return (EADDRINUSE);
 675                         } else if (t &&
 676                             ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 677                             (reuseport & inp_so_options(t)) == 0) {
 678 #ifdef INET6
 679                                 if (ntohl(sin->sin_addr.s_addr) !=
 680                                     INADDR_ANY ||
 681                                     ntohl(t->inp_laddr.s_addr) !=
 682                                     INADDR_ANY ||
 683                                     (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
 684                                     (t->inp_vflag & INP_IPV6PROTO) == 0)
 685 #endif
 686                                 return (EADDRINUSE);
 687                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 688                                         return (EADDRINUSE);
 689                         }
 690                 }
 691         }
 692         if (*lportp != 0)
 693                 lport = *lportp;
 694         if (lport == 0) {
 695                 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
 696                 if (error != 0)
 697                         return (error);
 698
 699         }
 700         *laddrp = laddr.s_addr;
 701         *lportp = lport;
 702         return (0);
 703 }
 704
 705 /*
 706  * Connect from a socket to a specified address.
 707  * Both address and port must be specified in argument sin.
 708  * If don't have a local address for this socket yet,
 709  * then pick one.
 710  */
 711 int
 712 in_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
 713     struct ucred *cred, struct mbuf *m)
 714 {
 715         u_short lport, fport;
 716         in_addr_t laddr, faddr;
 717         int anonport, error;
 718
 719         INP_WLOCK_ASSERT(inp);
 720         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 721
 722         lport = inp->inp_lport;
 723         laddr = inp->inp_laddr.s_addr;
 724         anonport = (lport == 0);
 725         error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport,
 726             NULL, cred);
 727         if (error)
 728                 return (error);
 729
 730         /* Do the initial binding of the local address if required. */
 731         if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
 732                 inp->inp_lport = lport;
 733                 inp->inp_laddr.s_addr = laddr;
 734                 if (in_pcbinshash(inp) != 0) {
 735                         inp->inp_laddr.s_addr = INADDR_ANY;
 736                         inp->inp_lport = 0;
 737                         return (EAGAIN);
 738                 }
 739         }
 740
 741         /* Commit the remaining changes. */
 742         inp->inp_lport = lport;
 743         inp->inp_laddr.s_addr = laddr;
 744         inp->inp_faddr.s_addr = faddr;
 745         inp->inp_fport = fport;
 746         in_pcbrehash_mbuf(inp, m);
 747
 748         if (anonport)
 749                 inp->inp_flags |= INP_ANONPORT;
 750         return (0);
 751 }
 752
 753 int
 754 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 755 {
 756
 757         return (in_pcbconnect_mbuf(inp, nam, cred, NULL));
 758 }
 759
 760 /*
 761  * Do proper source address selection on an unbound socket in case
 762  * of connect. Take jails into account as well.
 763  */
 764 int
 765 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
 766     struct ucred *cred)
 767 {
 768         struct ifaddr *ifa;
 769         struct sockaddr *sa;
 770         struct sockaddr_in *sin;
 771         struct route sro;
 772         int error;
 773
 774         KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
 775
 776         /*
 777          * Bypass source address selection and use the primary jail IP
 778          * if requested.
 779          */
 780         if (cred != NULL && !prison_saddrsel_ip4(cred, laddr))
 781                 return (0);
 782
 783         error = 0;
 784         bzero(&sro, sizeof(sro));
 785
 786         sin = (struct sockaddr_in *)&sro.ro_dst;
 787         sin->sin_family = AF_INET;
 788         sin->sin_len = sizeof(struct sockaddr_in);
 789         sin->sin_addr.s_addr = faddr->s_addr;
 790
 791         /*
 792          * If route is known our src addr is taken from the i/f,
 793          * else punt.
 794          *
 795          * Find out route to destination.
 796          */
 797         if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
 798                 in_rtalloc_ign(&sro, 0, inp->inp_inc.inc_fibnum);
 799
 800         /*
 801          * If we found a route, use the address corresponding to
 802          * the outgoing interface.
 803          *
 804          * Otherwise assume faddr is reachable on a directly connected
 805          * network and try to find a corresponding interface to take
 806          * the source address from.
 807          */
 808         if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) {
 809                 struct in_ifaddr *ia;
 810                 struct ifnet *ifp;
 811
 812                 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
 813                                         inp->inp_socket->so_fibnum));
 814                 if (ia == NULL)
 815                         ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
 816                                                 inp->inp_socket->so_fibnum));
 817                 if (ia == NULL) {
 818                         error = ENETUNREACH;
 819                         goto done;
 820                 }
 821
 822                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 823                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 824                         ifa_free(&ia->ia_ifa);
 825                         goto done;
 826                 }
 827
 828                 ifp = ia->ia_ifp;
 829                 ifa_free(&ia->ia_ifa);
 830                 ia = NULL;
 831                 IF_ADDR_RLOCK(ifp);
 832                 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 833
 834                         sa = ifa->ifa_addr;
 835                         if (sa->sa_family != AF_INET)
 836                                 continue;
 837                         sin = (struct sockaddr_in *)sa;
 838                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 839                                 ia = (struct in_ifaddr *)ifa;
 840                                 break;
 841                         }
 842                 }
 843                 if (ia != NULL) {
 844                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 845                         IF_ADDR_RUNLOCK(ifp);
 846                         goto done;
 847                 }
 848                 IF_ADDR_RUNLOCK(ifp);
 849
 850                 /* 3. As a last resort return the 'default' jail address. */
 851                 error = prison_get_ip4(cred, laddr);
 852                 goto done;
 853         }
 854
 855         /*
 856          * If the outgoing interface on the route found is not
 857          * a loopback interface, use the address from that interface.
 858          * In case of jails do those three steps:
 859          * 1. check if the interface address belongs to the jail. If so use it.
 860          * 2. check if we have any address on the outgoing interface
 861          *    belonging to this jail. If so use it.
 862          * 3. as a last resort return the 'default' jail address.
 863          */
 864         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) == 0) {
 865                 struct in_ifaddr *ia;
 866                 struct ifnet *ifp;
 867
 868                 /* If not jailed, use the default returned. */
 869                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 870                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 871                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 872                         goto done;
 873                 }
 874
 875                 /* Jailed. */
 876                 /* 1. Check if the iface address belongs to the jail. */
 877                 sin = (struct sockaddr_in *)sro.ro_rt->rt_ifa->ifa_addr;
 878                 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 879                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
 880                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 881                         goto done;
 882                 }
 883
 884                 /*
 885                  * 2. Check if we have any address on the outgoing interface
 886                  *    belonging to this jail.
 887                  */
 888                 ia = NULL;
 889                 ifp = sro.ro_rt->rt_ifp;
 890                 IF_ADDR_RLOCK(ifp);
 891                 TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 892                         sa = ifa->ifa_addr;
 893                         if (sa->sa_family != AF_INET)
 894                                 continue;
 895                         sin = (struct sockaddr_in *)sa;
 896                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
 897                                 ia = (struct in_ifaddr *)ifa;
 898                                 break;
 899                         }
 900                 }
 901                 if (ia != NULL) {
 902                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 903                         IF_ADDR_RUNLOCK(ifp);
 904                         goto done;
 905                 }
 906                 IF_ADDR_RUNLOCK(ifp);
 907
 908                 /* 3. As a last resort return the 'default' jail address. */
 909                 error = prison_get_ip4(cred, laddr);
 910                 goto done;
 911         }
 912
 913         /*
 914          * The outgoing interface is marked with 'loopback net', so a route
 915          * to ourselves is here.
 916          * Try to find the interface of the destination address and then
 917          * take the address from there. That interface is not necessarily
 918          * a loopback interface.
 919          * In case of jails, check that it is an address of the jail
 920          * and if we cannot find, fall back to the 'default' jail address.
 921          */
 922         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
 923                 struct sockaddr_in sain;
 924                 struct in_ifaddr *ia;
 925
 926                 bzero(&sain, sizeof(struct sockaddr_in));
 927                 sain.sin_family = AF_INET;
 928                 sain.sin_len = sizeof(struct sockaddr_in);
 929                 sain.sin_addr.s_addr = faddr->s_addr;
 930
 931                 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&sain),
 932                                         inp->inp_socket->so_fibnum));
 933                 if (ia == NULL)
 934                         ia = ifatoia(ifa_ifwithnet(sintosa(&sain), 0,
 935                                                 inp->inp_socket->so_fibnum));
 936                 if (ia == NULL)
 937                         ia = ifatoia(ifa_ifwithaddr(sintosa(&sain)));
 938
 939                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
 940                         if (ia == NULL) {
 941                                 error = ENETUNREACH;
 942                                 goto done;
 943                         }
 944                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 945                         ifa_free(&ia->ia_ifa);
 946                         goto done;
 947                 }
 948
 949                 /* Jailed. */
 950                 if (ia != NULL) {
 951                         struct ifnet *ifp;
 952
 953                         ifp = ia->ia_ifp;
 954                         ifa_free(&ia->ia_ifa);
 955                         ia = NULL;
 956                         IF_ADDR_RLOCK(ifp);
 957                         TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 958
 959                                 sa = ifa->ifa_addr;
 960                                 if (sa->sa_family != AF_INET)
 961                                         continue;
 962                                 sin = (struct sockaddr_in *)sa;
 963                                 if (prison_check_ip4(cred,
 964                                     &sin->sin_addr) == 0) {
 965                                         ia = (struct in_ifaddr *)ifa;
 966                                         break;
 967                                 }
 968                         }
 969                         if (ia != NULL) {
 970                                 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
 971                                 IF_ADDR_RUNLOCK(ifp);
 972                                 goto done;
 973                         }
 974                         IF_ADDR_RUNLOCK(ifp);
 975                 }
 976
 977                 /* 3. As a last resort return the 'default' jail address. */
 978                 error = prison_get_ip4(cred, laddr);
 979                 goto done;
 980         }
 981
 982 done:
 983         if (sro.ro_rt != NULL)
 984                 RTFREE(sro.ro_rt);
 985         return (error);
 986 }
 987
 988 /*
 989  * Set up for a connect from a socket to the specified address.
 990  * On entry, *laddrp and *lportp should contain the current local
 991  * address and port for the PCB; these are updated to the values
 992  * that should be placed in inp_laddr and inp_lport to complete
 993  * the connect.
 994  *
 995  * On success, *faddrp and *fportp will be set to the remote address
 996  * and port. These are not updated in the error case.
 997  *
 998  * If the operation fails because the connection already exists,
 999  * *oinpp will be set to the PCB of that connection so that the
1000  * caller can decide to override it. In all other cases, *oinpp
1001  * is set to NULL.
1002  */
1003 int
1004 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
1005     in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1006     struct inpcb **oinpp, struct ucred *cred)
1007 {
1008         struct rm_priotracker in_ifa_tracker;
1009         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1010         struct in_ifaddr *ia;
1011         struct inpcb *oinp;
1012         struct in_addr laddr, faddr;
1013         u_short lport, fport;
1014         int error;
1015
1016         /*
1017          * Because a global state change doesn't actually occur here, a read
1018          * lock is sufficient.
1019          */
1020         INP_LOCK_ASSERT(inp);
1021         INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1022
1023         if (oinpp != NULL)
1024                 *oinpp = NULL;
1025         if (nam->sa_len != sizeof (*sin))
1026                 return (EINVAL);
1027         if (sin->sin_family != AF_INET)
1028                 return (EAFNOSUPPORT);
1029         if (sin->sin_port == 0)
1030                 return (EADDRNOTAVAIL);
1031         laddr.s_addr = *laddrp;
1032         lport = *lportp;
1033         faddr = sin->sin_addr;
1034         fport = sin->sin_port;
1035
1036         if (!TAILQ_EMPTY(&V_in_ifaddrhead)) {
1037                 /*
1038                  * If the destination address is INADDR_ANY,
1039                  * use the primary local address.
1040                  * If the supplied address is INADDR_BROADCAST,
1041                  * and the primary interface supports broadcast,
1042                  * choose the broadcast address for that interface.
1043                  */
1044                 if (faddr.s_addr == INADDR_ANY) {
1045                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1046                         faddr =
1047                             IA_SIN(TAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1048                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1049                         if (cred != NULL &&
1050                             (error = prison_get_ip4(cred, &faddr)) != 0)
1051                                 return (error);
1052                 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1053                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1054                         if (TAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1055                             IFF_BROADCAST)
1056                                 faddr = satosin(&TAILQ_FIRST(
1057                                     &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1058                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1059                 }
1060         }
1061         if (laddr.s_addr == INADDR_ANY) {
1062                 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1063                 /*
1064                  * If the destination address is multicast and an outgoing
1065                  * interface has been set as a multicast option, prefer the
1066                  * address of that interface as our source address.
1067                  */
1068                 if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1069                     inp->inp_moptions != NULL) {
1070                         struct ip_moptions *imo;
1071                         struct ifnet *ifp;
1072
1073                         imo = inp->inp_moptions;
1074                         if (imo->imo_multicast_ifp != NULL) {
1075                                 ifp = imo->imo_multicast_ifp;
1076                                 IN_IFADDR_RLOCK(&in_ifa_tracker);
1077                                 TAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1078                                         if ((ia->ia_ifp == ifp) &&
1079                                             (cred == NULL ||
1080                                             prison_check_ip4(cred,
1081                                             &ia->ia_addr.sin_addr) == 0))
1082                                                 break;
1083                                 }
1084                                 if (ia == NULL)
1085                                         error = EADDRNOTAVAIL;
1086                                 else {
1087                                         laddr = ia->ia_addr.sin_addr;
1088                                         error = 0;
1089                                 }
1090                                 IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1091                         }
1092                 }
1093                 if (error)
1094                         return (error);
1095         }
1096         oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, fport,
1097             laddr, lport, 0, NULL);
1098         if (oinp != NULL) {
1099                 if (oinpp != NULL)
1100                         *oinpp = oinp;
1101                 return (EADDRINUSE);
1102         }
1103         if (lport == 0) {
1104                 error = in_pcbbind_setup(inp, NULL, &laddr.s_addr, &lport,
1105                     cred);
1106                 if (error)
1107                         return (error);
1108         }
1109         *laddrp = laddr.s_addr;
1110         *lportp = lport;
1111         *faddrp = faddr.s_addr;
1112         *fportp = fport;
1113         return (0);
1114 }
1115
1116 void
1117 in_pcbdisconnect(struct inpcb *inp)
1118 {
1119
1120         INP_WLOCK_ASSERT(inp);
1121         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1122
1123         inp->inp_faddr.s_addr = INADDR_ANY;
1124         inp->inp_fport = 0;
1125         in_pcbrehash(inp);
1126 }
1127 #endif /* INET */
1128
1129 /*
1130  * in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
1131  * For most protocols, this will be invoked immediately prior to calling
1132  * in_pcbfree().  However, with TCP the inpcb may significantly outlive the
1133  * socket, in which case in_pcbfree() is deferred.
1134  */
1135 void
1136 in_pcbdetach(struct inpcb *inp)
1137 {
1138
1139         KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1140
1141 #ifdef RATELIMIT
1142         if (inp->inp_snd_tag != NULL)
1143                 in_pcbdetach_txrtlmt(inp);
1144 #endif
1145         inp->inp_socket->so_pcb = NULL;
1146         inp->inp_socket = NULL;
1147 }
1148
1149 /*
1150  * in_pcbref() bumps the reference count on an inpcb in order to maintain
1151  * stability of an inpcb pointer despite the inpcb lock being released.  This
1152  * is used in TCP when the inpcbinfo lock needs to be acquired or upgraded,
1153  * but where the inpcb lock may already held, or when acquiring a reference
1154  * via a pcbgroup.
1155  *
1156  * in_pcbref() should be used only to provide brief memory stability, and
1157  * must always be followed by a call to INP_WLOCK() and in_pcbrele() to
1158  * garbage collect the inpcb if it has been in_pcbfree()'d from another
1159  * context.  Until in_pcbrele() has returned that the inpcb is still valid,
1160  * lock and rele are the *only* safe operations that may be performed on the
1161  * inpcb.
1162  *
1163  * While the inpcb will not be freed, releasing the inpcb lock means that the
1164  * connection's state may change, so the caller should be careful to
1165  * revalidate any cached state on reacquiring the lock.  Drop the reference
1166  * using in_pcbrele().
1167  */
1168 void
1169 in_pcbref(struct inpcb *inp)
1170 {
1171
1172         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1173
1174         refcount_acquire(&inp->inp_refcount);
1175 }
1176
1177 /*
1178  * Drop a refcount on an inpcb elevated using in_pcbref(); because a call to
1179  * in_pcbfree() may have been made between in_pcbref() and in_pcbrele(), we
1180  * return a flag indicating whether or not the inpcb remains valid.  If it is
1181  * valid, we return with the inpcb lock held.
1182  *
1183  * Notice that, unlike in_pcbref(), the inpcb lock must be held to drop a
1184  * reference on an inpcb.  Historically more work was done here (actually, in
1185  * in_pcbfree_internal()) but has been moved to in_pcbfree() to avoid the
1186  * need for the pcbinfo lock in in_pcbrele().  Deferring the free is entirely
1187  * about memory stability (and continued use of the write lock).
1188  */
1189 int
1190 in_pcbrele_rlocked(struct inpcb *inp)
1191 {
1192         struct inpcbinfo *pcbinfo;
1193
1194         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1195
1196         INP_RLOCK_ASSERT(inp);
1197
1198         if (refcount_release(&inp->inp_refcount) == 0) {
1199                 /*
1200                  * If the inpcb has been freed, let the caller know, even if
1201                  * this isn't the last reference.
1202                  */
1203                 if (inp->inp_flags2 & INP_FREED) {
1204                         INP_RUNLOCK(inp);
1205                         return (1);
1206                 }
1207                 return (0);
1208         }
1209
1210         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1211
1212         INP_RUNLOCK(inp);
1213         pcbinfo = inp->inp_pcbinfo;
1214         uma_zfree(pcbinfo->ipi_zone, inp);
1215         return (1);
1216 }
1217
1218 int
1219 in_pcbrele_wlocked(struct inpcb *inp)
1220 {
1221         struct inpcbinfo *pcbinfo;
1222
1223         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1224
1225         INP_WLOCK_ASSERT(inp);
1226
1227         if (refcount_release(&inp->inp_refcount) == 0) {
1228                 /*
1229                  * If the inpcb has been freed, let the caller know, even if
1230                  * this isn't the last reference.
1231                  */
1232                 if (inp->inp_flags2 & INP_FREED) {
1233                         INP_WUNLOCK(inp);
1234                         return (1);
1235                 }
1236                 return (0);
1237         }
1238
1239         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1240
1241         INP_WUNLOCK(inp);
1242         pcbinfo = inp->inp_pcbinfo;
1243         uma_zfree(pcbinfo->ipi_zone, inp);
1244         return (1);
1245 }
1246
1247 /*
1248  * Temporary wrapper.
1249  */
1250 int
1251 in_pcbrele(struct inpcb *inp)
1252 {
1253
1254         return (in_pcbrele_wlocked(inp));
1255 }
1256
1257 /*
1258  * Unconditionally schedule an inpcb to be freed by decrementing its
1259  * reference count, which should occur only after the inpcb has been detached
1260  * from its socket.  If another thread holds a temporary reference (acquired
1261  * using in_pcbref()) then the free is deferred until that reference is
1262  * released using in_pcbrele(), but the inpcb is still unlocked.  Almost all
1263  * work, including removal from global lists, is done in this context, where
1264  * the pcbinfo lock is held.
1265  */
1266 void
1267 in_pcbfree(struct inpcb *inp)
1268 {
1269         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1270
1271         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1272
1273 #ifdef INVARIANTS
1274         if (pcbinfo == &V_tcbinfo) {
1275                 INP_INFO_LOCK_ASSERT(pcbinfo);
1276         } else {
1277                 INP_INFO_WLOCK_ASSERT(pcbinfo);
1278         }
1279 #endif
1280         INP_WLOCK_ASSERT(inp);
1281
1282         /* XXXRW: Do as much as possible here. */
1283 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1284         if (inp->inp_sp != NULL)
1285                 ipsec_delete_pcbpolicy(inp);
1286 #endif
1287         INP_LIST_WLOCK(pcbinfo);
1288         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
1289         in_pcbremlists(inp);
1290         INP_LIST_WUNLOCK(pcbinfo);
1291 #ifdef INET6
1292         if (inp->inp_vflag & INP_IPV6PROTO) {
1293                 ip6_freepcbopts(inp->in6p_outputopts);
1294                 if (inp->in6p_moptions != NULL)
1295                         ip6_freemoptions(inp->in6p_moptions);
1296         }
1297 #endif
1298         if (inp->inp_options)
1299                 (void)m_free(inp->inp_options);
1300 #ifdef INET
1301         if (inp->inp_moptions != NULL)
1302                 inp_freemoptions(inp->inp_moptions);
1303 #endif
1304         RO_RTFREE(&inp->inp_route);
1305         if (inp->inp_route.ro_lle)
1306                 LLE_FREE(inp->inp_route.ro_lle);        /* zeros ro_lle */
1307
1308         inp->inp_vflag = 0;
1309         inp->inp_flags2 |= INP_FREED;
1310         crfree(inp->inp_cred);
1311 #ifdef MAC
1312         mac_inpcb_destroy(inp);
1313 #endif
1314         if (!in_pcbrele_wlocked(inp))
1315                 INP_WUNLOCK(inp);
1316 }
1317
1318 /*
1319  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1320  * port reservation, and preventing it from being returned by inpcb lookups.
1321  *
1322  * It is used by TCP to mark an inpcb as unused and avoid future packet
1323  * delivery or event notification when a socket remains open but TCP has
1324  * closed.  This might occur as a result of a shutdown()-initiated TCP close
1325  * or a RST on the wire, and allows the port binding to be reused while still
1326  * maintaining the invariant that so_pcb always points to a valid inpcb until
1327  * in_pcbdetach().
1328  *
1329  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1330  * in_pcbnotifyall() and in_pcbpurgeif0()?
1331  */
1332 void
1333 in_pcbdrop(struct inpcb *inp)
1334 {
1335
1336         INP_WLOCK_ASSERT(inp);
1337
1338         /*
1339          * XXXRW: Possibly we should protect the setting of INP_DROPPED with
1340          * the hash lock...?
1341          */
1342         inp->inp_flags |= INP_DROPPED;
1343         if (inp->inp_flags & INP_INHASHLIST) {
1344                 struct inpcbport *phd = inp->inp_phd;
1345
1346                 INP_HASH_WLOCK(inp->inp_pcbinfo);
1347                 LIST_REMOVE(inp, inp_hash);
1348                 LIST_REMOVE(inp, inp_portlist);
1349                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
1350                         LIST_REMOVE(phd, phd_hash);
1351                         free(phd, M_PCB);
1352                 }
1353                 INP_HASH_WUNLOCK(inp->inp_pcbinfo);
1354                 inp->inp_flags &= ~INP_INHASHLIST;
1355 #ifdef PCBGROUP
1356                 in_pcbgroup_remove(inp);
1357 #endif
1358         }
1359 }
1360
1361 #ifdef INET
1362 /*
1363  * Common routines to return the socket addresses associated with inpcbs.
1364  */
1365 struct sockaddr *
1366 in_sockaddr(in_port_t port, struct in_addr *addr_p)
1367 {
1368         struct sockaddr_in *sin;
1369
1370         sin = malloc(sizeof *sin, M_SONAME,
1371                 M_WAITOK | M_ZERO);
1372         sin->sin_family = AF_INET;
1373         sin->sin_len = sizeof(*sin);
1374         sin->sin_addr = *addr_p;
1375         sin->sin_port = port;
1376
1377         return (struct sockaddr *)sin;
1378 }
1379
1380 int
1381 in_getsockaddr(struct socket *so, struct sockaddr **nam)
1382 {
1383         struct inpcb *inp;
1384         struct in_addr addr;
1385         in_port_t port;
1386
1387         inp = sotoinpcb(so);
1388         KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1389
1390         INP_RLOCK(inp);
1391         port = inp->inp_lport;
1392         addr = inp->inp_laddr;
1393         INP_RUNLOCK(inp);
1394
1395         *nam = in_sockaddr(port, &addr);
1396         return 0;
1397 }
1398
1399 int
1400 in_getpeeraddr(struct socket *so, struct sockaddr **nam)
1401 {
1402         struct inpcb *inp;
1403         struct in_addr addr;
1404         in_port_t port;
1405
1406         inp = sotoinpcb(so);
1407         KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1408
1409         INP_RLOCK(inp);
1410         port = inp->inp_fport;
1411         addr = inp->inp_faddr;
1412         INP_RUNLOCK(inp);
1413
1414         *nam = in_sockaddr(port, &addr);
1415         return 0;
1416 }
1417
1418 void
1419 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
1420     struct inpcb *(*notify)(struct inpcb *, int))
1421 {
1422         struct inpcb *inp, *inp_temp;
1423
1424         INP_INFO_WLOCK(pcbinfo);
1425         LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
1426                 INP_WLOCK(inp);
1427 #ifdef INET6
1428                 if ((inp->inp_vflag & INP_IPV4) == 0) {
1429                         INP_WUNLOCK(inp);
1430                         continue;
1431                 }
1432 #endif
1433                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1434                     inp->inp_socket == NULL) {
1435                         INP_WUNLOCK(inp);
1436                         continue;
1437                 }
1438                 if ((*notify)(inp, errno))
1439                         INP_WUNLOCK(inp);
1440         }
1441         INP_INFO_WUNLOCK(pcbinfo);
1442 }
1443
1444 void
1445 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1446 {
1447         struct inpcb *inp;
1448         struct ip_moptions *imo;
1449         int i, gap;
1450
1451         INP_INFO_WLOCK(pcbinfo);
1452         LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1453                 INP_WLOCK(inp);
1454                 imo = inp->inp_moptions;
1455                 if ((inp->inp_vflag & INP_IPV4) &&
1456                     imo != NULL) {
1457                         /*
1458                          * Unselect the outgoing interface if it is being
1459                          * detached.
1460                          */
1461                         if (imo->imo_multicast_ifp == ifp)
1462                                 imo->imo_multicast_ifp = NULL;
1463
1464                         /*
1465                          * Drop multicast group membership if we joined
1466                          * through the interface being detached.
1467                          */
1468                         for (i = 0, gap = 0; i < imo->imo_num_memberships;
1469                             i++) {
1470                                 if (imo->imo_membership[i]->inm_ifp == ifp) {
1471                                         in_delmulti(imo->imo_membership[i]);
1472                                         gap++;
1473                                 } else if (gap != 0)
1474                                         imo->imo_membership[i - gap] =
1475                                             imo->imo_membership[i];
1476                         }
1477                         imo->imo_num_memberships -= gap;
1478                 }
1479                 INP_WUNLOCK(inp);
1480         }
1481         INP_INFO_WUNLOCK(pcbinfo);
1482 }
1483
1484 /*
1485  * Lookup a PCB based on the local address and port.  Caller must hold the
1486  * hash lock.  No inpcb locks or references are acquired.
1487  */
1488 #define INP_LOOKUP_MAPPED_PCB_COST      3
1489 struct inpcb *
1490 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1491     u_short lport, int lookupflags, struct ucred *cred)
1492 {
1493         struct inpcb *inp;
1494 #ifdef INET6
1495         int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1496 #else
1497         int matchwild = 3;
1498 #endif
1499         int wildcard;
1500
1501         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1502             ("%s: invalid lookup flags %d", __func__, lookupflags));
1503
1504         INP_HASH_LOCK_ASSERT(pcbinfo);
1505
1506         if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1507                 struct inpcbhead *head;
1508                 /*
1509                  * Look for an unconnected (wildcard foreign addr) PCB that
1510                  * matches the local address and port we're looking for.
1511                  */
1512                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1513                     0, pcbinfo->ipi_hashmask)];
1514                 LIST_FOREACH(inp, head, inp_hash) {
1515 #ifdef INET6
1516                         /* XXX inp locking */
1517                         if ((inp->inp_vflag & INP_IPV4) == 0)
1518                                 continue;
1519 #endif
1520                         if (inp->inp_faddr.s_addr == INADDR_ANY &&
1521                             inp->inp_laddr.s_addr == laddr.s_addr &&
1522                             inp->inp_lport == lport) {
1523                                 /*
1524                                  * Found?
1525                                  */
1526                                 if (cred == NULL ||
1527                                     prison_equal_ip4(cred->cr_prison,
1528                                         inp->inp_cred->cr_prison))
1529                                         return (inp);
1530                         }
1531                 }
1532                 /*
1533                  * Not found.
1534                  */
1535                 return (NULL);
1536         } else {
1537                 struct inpcbporthead *porthash;
1538                 struct inpcbport *phd;
1539                 struct inpcb *match = NULL;
1540                 /*
1541                  * Best fit PCB lookup.
1542                  *
1543                  * First see if this local port is in use by looking on the
1544                  * port hash list.
1545                  */
1546                 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
1547                     pcbinfo->ipi_porthashmask)];
1548                 LIST_FOREACH(phd, porthash, phd_hash) {
1549                         if (phd->phd_port == lport)
1550                                 break;
1551                 }
1552                 if (phd != NULL) {
1553                         /*
1554                          * Port is in use by one or more PCBs. Look for best
1555                          * fit.
1556                          */
1557                         LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1558                                 wildcard = 0;
1559                                 if (cred != NULL &&
1560                                     !prison_equal_ip4(inp->inp_cred->cr_prison,
1561                                         cred->cr_prison))
1562                                         continue;
1563 #ifdef INET6
1564                                 /* XXX inp locking */
1565                                 if ((inp->inp_vflag & INP_IPV4) == 0)
1566                                         continue;
1567                                 /*
1568                                  * We never select the PCB that has
1569                                  * INP_IPV6 flag and is bound to :: if
1570                                  * we have another PCB which is bound
1571                                  * to 0.0.0.0.  If a PCB has the
1572                                  * INP_IPV6 flag, then we set its cost
1573                                  * higher than IPv4 only PCBs.
1574                                  *
1575                                  * Note that the case only happens
1576                                  * when a socket is bound to ::, under
1577                                  * the condition that the use of the
1578                                  * mapped address is allowed.
1579                                  */
1580                                 if ((inp->inp_vflag & INP_IPV6) != 0)
1581                                         wildcard += INP_LOOKUP_MAPPED_PCB_COST;
1582 #endif
1583                                 if (inp->inp_faddr.s_addr != INADDR_ANY)
1584                                         wildcard++;
1585                                 if (inp->inp_laddr.s_addr != INADDR_ANY) {
1586                                         if (laddr.s_addr == INADDR_ANY)
1587                                                 wildcard++;
1588                                         else if (inp->inp_laddr.s_addr != laddr.s_addr)
1589                                                 continue;
1590                                 } else {
1591                                         if (laddr.s_addr != INADDR_ANY)
1592                                                 wildcard++;
1593                                 }
1594                                 if (wildcard < matchwild) {
1595                                         match = inp;
1596                                         matchwild = wildcard;
1597                                         if (matchwild == 0)
1598                                                 break;
1599                                 }
1600                         }
1601                 }
1602                 return (match);
1603         }
1604 }
1605 #undef INP_LOOKUP_MAPPED_PCB_COST
1606
1607 #ifdef PCBGROUP
1608 /*
1609  * Lookup PCB in hash list, using pcbgroup tables.
1610  */
1611 static struct inpcb *
1612 in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
1613     struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
1614     u_int lport_arg, int lookupflags, struct ifnet *ifp)
1615 {
1616         struct inpcbhead *head;
1617         struct inpcb *inp, *tmpinp;
1618         u_short fport = fport_arg, lport = lport_arg;
1619
1620         /*
1621          * First look for an exact match.
1622          */
1623         tmpinp = NULL;
1624         INP_GROUP_LOCK(pcbgroup);
1625         head = &pcbgroup->ipg_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
1626             pcbgroup->ipg_hashmask)];
1627         LIST_FOREACH(inp, head, inp_pcbgrouphash) {
1628 #ifdef INET6
1629                 /* XXX inp locking */
1630                 if ((inp->inp_vflag & INP_IPV4) == 0)
1631                         continue;
1632 #endif
1633                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
1634                     inp->inp_laddr.s_addr == laddr.s_addr &&
1635                     inp->inp_fport == fport &&
1636                     inp->inp_lport == lport) {
1637                         /*
1638                          * XXX We should be able to directly return
1639                          * the inp here, without any checks.
1640                          * Well unless both bound with SO_REUSEPORT?
1641                          */
1642                         if (prison_flag(inp->inp_cred, PR_IP4))
1643                                 goto found;
1644                         if (tmpinp == NULL)
1645                                 tmpinp = inp;
1646                 }
1647         }
1648         if (tmpinp != NULL) {
1649                 inp = tmpinp;
1650                 goto found;
1651         }
1652
1653 #ifdef  RSS
1654         /*
1655          * For incoming connections, we may wish to do a wildcard
1656          * match for an RSS-local socket.
1657          */
1658         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1659                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1660 #ifdef INET6
1661                 struct inpcb *local_wild_mapped = NULL;
1662 #endif
1663                 struct inpcb *jail_wild = NULL;
1664                 struct inpcbhead *head;
1665                 int injail;
1666
1667                 /*
1668                  * Order of socket selection - we always prefer jails.
1669                  *      1. jailed, non-wild.
1670                  *      2. jailed, wild.
1671                  *      3. non-jailed, non-wild.
1672                  *      4. non-jailed, wild.
1673                  */
1674
1675                 head = &pcbgroup->ipg_hashbase[INP_PCBHASH(INADDR_ANY,
1676                     lport, 0, pcbgroup->ipg_hashmask)];
1677                 LIST_FOREACH(inp, head, inp_pcbgrouphash) {
1678 #ifdef INET6
1679                         /* XXX inp locking */
1680                         if ((inp->inp_vflag & INP_IPV4) == 0)
1681                                 continue;
1682 #endif
1683                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1684                             inp->inp_lport != lport)
1685                                 continue;
1686
1687                         injail = prison_flag(inp->inp_cred, PR_IP4);
1688                         if (injail) {
1689                                 if (prison_check_ip4(inp->inp_cred,
1690                                     &laddr) != 0)
1691                                         continue;
1692                         } else {
1693                                 if (local_exact != NULL)
1694                                         continue;
1695                         }
1696
1697                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1698                                 if (injail)
1699                                         goto found;
1700                                 else
1701                                         local_exact = inp;
1702                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1703 #ifdef INET6
1704                                 /* XXX inp locking, NULL check */
1705                                 if (inp->inp_vflag & INP_IPV6PROTO)
1706                                         local_wild_mapped = inp;
1707                                 else
1708 #endif
1709                                         if (injail)
1710                                                 jail_wild = inp;
1711                                         else
1712                                                 local_wild = inp;
1713                         }
1714                 } /* LIST_FOREACH */
1715
1716                 inp = jail_wild;
1717                 if (inp == NULL)
1718                         inp = local_exact;
1719                 if (inp == NULL)
1720                         inp = local_wild;
1721 #ifdef INET6
1722                 if (inp == NULL)
1723                         inp = local_wild_mapped;
1724 #endif
1725                 if (inp != NULL)
1726                         goto found;
1727         }
1728 #endif
1729
1730         /*
1731          * Then look for a wildcard match, if requested.
1732          */
1733         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1734                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1735 #ifdef INET6
1736                 struct inpcb *local_wild_mapped = NULL;
1737 #endif
1738                 struct inpcb *jail_wild = NULL;
1739                 struct inpcbhead *head;
1740                 int injail;
1741
1742                 /*
1743                  * Order of socket selection - we always prefer jails.
1744                  *      1. jailed, non-wild.
1745                  *      2. jailed, wild.
1746                  *      3. non-jailed, non-wild.
1747                  *      4. non-jailed, wild.
1748                  */
1749                 head = &pcbinfo->ipi_wildbase[INP_PCBHASH(INADDR_ANY, lport,
1750                     0, pcbinfo->ipi_wildmask)];
1751                 LIST_FOREACH(inp, head, inp_pcbgroup_wild) {
1752 #ifdef INET6
1753                         /* XXX inp locking */
1754                         if ((inp->inp_vflag & INP_IPV4) == 0)
1755                                 continue;
1756 #endif
1757                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1758                             inp->inp_lport != lport)
1759                                 continue;
1760
1761                         injail = prison_flag(inp->inp_cred, PR_IP4);
1762                         if (injail) {
1763                                 if (prison_check_ip4(inp->inp_cred,
1764                                     &laddr) != 0)
1765                                         continue;
1766                         } else {
1767                                 if (local_exact != NULL)
1768                                         continue;
1769                         }
1770
1771                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1772                                 if (injail)
1773                                         goto found;
1774                                 else
1775                                         local_exact = inp;
1776                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1777 #ifdef INET6
1778                                 /* XXX inp locking, NULL check */
1779                                 if (inp->inp_vflag & INP_IPV6PROTO)
1780                                         local_wild_mapped = inp;
1781                                 else
1782 #endif
1783                                         if (injail)
1784                                                 jail_wild = inp;
1785                                         else
1786                                                 local_wild = inp;
1787                         }
1788                 } /* LIST_FOREACH */
1789                 inp = jail_wild;
1790                 if (inp == NULL)
1791                         inp = local_exact;
1792                 if (inp == NULL)
1793                         inp = local_wild;
1794 #ifdef INET6
1795                 if (inp == NULL)
1796                         inp = local_wild_mapped;
1797 #endif
1798                 if (inp != NULL)
1799                         goto found;
1800         } /* if (lookupflags & INPLOOKUP_WILDCARD) */
1801         INP_GROUP_UNLOCK(pcbgroup);
1802         return (NULL);
1803
1804 found:
1805         in_pcbref(inp);
1806         INP_GROUP_UNLOCK(pcbgroup);
1807         if (lookupflags & INPLOOKUP_WLOCKPCB) {
1808                 INP_WLOCK(inp);
1809                 if (in_pcbrele_wlocked(inp))
1810                         return (NULL);
1811         } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
1812                 INP_RLOCK(inp);
1813                 if (in_pcbrele_rlocked(inp))
1814                         return (NULL);
1815         } else
1816                 panic("%s: locking bug", __func__);
1817         return (inp);
1818 }
1819 #endif /* PCBGROUP */
1820
1821 /*
1822  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
1823  * that the caller has locked the hash list, and will not perform any further
1824  * locking or reference operations on either the hash list or the connection.
1825  */
1826 static struct inpcb *
1827 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1828     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
1829     struct ifnet *ifp)
1830 {
1831         struct inpcbhead *head;
1832         struct inpcb *inp, *tmpinp;
1833         u_short fport = fport_arg, lport = lport_arg;
1834
1835         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1836             ("%s: invalid lookup flags %d", __func__, lookupflags));
1837
1838         INP_HASH_LOCK_ASSERT(pcbinfo);
1839
1840         /*
1841          * First look for an exact match.
1842          */
1843         tmpinp = NULL;
1844         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
1845             pcbinfo->ipi_hashmask)];
1846         LIST_FOREACH(inp, head, inp_hash) {
1847 #ifdef INET6
1848                 /* XXX inp locking */
1849                 if ((inp->inp_vflag & INP_IPV4) == 0)
1850                         continue;
1851 #endif
1852                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
1853                     inp->inp_laddr.s_addr == laddr.s_addr &&
1854                     inp->inp_fport == fport &&
1855                     inp->inp_lport == lport) {
1856                         /*
1857                          * XXX We should be able to directly return
1858                          * the inp here, without any checks.
1859                          * Well unless both bound with SO_REUSEPORT?
1860                          */
1861                         if (prison_flag(inp->inp_cred, PR_IP4))
1862                                 return (inp);
1863                         if (tmpinp == NULL)
1864                                 tmpinp = inp;
1865                 }
1866         }
1867         if (tmpinp != NULL)
1868                 return (tmpinp);
1869
1870         /*
1871          * Then look for a wildcard match, if requested.
1872          */
1873         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1874                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1875 #ifdef INET6
1876                 struct inpcb *local_wild_mapped = NULL;
1877 #endif
1878                 struct inpcb *jail_wild = NULL;
1879                 int injail;
1880
1881                 /*
1882                  * Order of socket selection - we always prefer jails.
1883                  *      1. jailed, non-wild.
1884                  *      2. jailed, wild.
1885                  *      3. non-jailed, non-wild.
1886                  *      4. non-jailed, wild.
1887                  */
1888
1889                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1890                     0, pcbinfo->ipi_hashmask)];
1891                 LIST_FOREACH(inp, head, inp_hash) {
1892 #ifdef INET6
1893                         /* XXX inp locking */
1894                         if ((inp->inp_vflag & INP_IPV4) == 0)
1895                                 continue;
1896 #endif
1897                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
1898                             inp->inp_lport != lport)
1899                                 continue;
1900
1901                         injail = prison_flag(inp->inp_cred, PR_IP4);
1902                         if (injail) {
1903                                 if (prison_check_ip4(inp->inp_cred,
1904                                     &laddr) != 0)
1905                                         continue;
1906                         } else {
1907                                 if (local_exact != NULL)
1908                                         continue;
1909                         }
1910
1911                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
1912                                 if (injail)
1913                                         return (inp);
1914                                 else
1915                                         local_exact = inp;
1916                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
1917 #ifdef INET6
1918                                 /* XXX inp locking, NULL check */
1919                                 if (inp->inp_vflag & INP_IPV6PROTO)
1920                                         local_wild_mapped = inp;
1921                                 else
1922 #endif
1923                                         if (injail)
1924                                                 jail_wild = inp;
1925                                         else
1926                                                 local_wild = inp;
1927                         }
1928                 } /* LIST_FOREACH */
1929                 if (jail_wild != NULL)
1930                         return (jail_wild);
1931                 if (local_exact != NULL)
1932                         return (local_exact);
1933                 if (local_wild != NULL)
1934                         return (local_wild);
1935 #ifdef INET6
1936                 if (local_wild_mapped != NULL)
1937                         return (local_wild_mapped);
1938 #endif
1939         } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
1940
1941         return (NULL);
1942 }
1943
1944 /*
1945  * Lookup PCB in hash list, using pcbinfo tables.  This variation locks the
1946  * hash list lock, and will return the inpcb locked (i.e., requires
1947  * INPLOOKUP_LOCKPCB).
1948  */
1949 static struct inpcb *
1950 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
1951     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
1952     struct ifnet *ifp)
1953 {
1954         struct inpcb *inp;
1955
1956         INP_HASH_RLOCK(pcbinfo);
1957         inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
1958             (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
1959         if (inp != NULL) {
1960                 in_pcbref(inp);
1961                 INP_HASH_RUNLOCK(pcbinfo);
1962                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
1963                         INP_WLOCK(inp);
1964                         if (in_pcbrele_wlocked(inp))
1965                                 return (NULL);
1966                 } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
1967                         INP_RLOCK(inp);
1968                         if (in_pcbrele_rlocked(inp))
1969                                 return (NULL);
1970                 } else
1971                         panic("%s: locking bug", __func__);
1972         } else
1973                 INP_HASH_RUNLOCK(pcbinfo);
1974         return (inp);
1975 }
1976
1977 /*
1978  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
1979  * from which a pre-calculated hash value may be extracted.
1980  *
1981  * Possibly more of this logic should be in in_pcbgroup.c.
1982  */
1983 struct inpcb *
1984 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
1985     struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp)
1986 {
1987 #if defined(PCBGROUP) && !defined(RSS)
1988         struct inpcbgroup *pcbgroup;
1989 #endif
1990
1991         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
1992             ("%s: invalid lookup flags %d", __func__, lookupflags));
1993         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
1994             ("%s: LOCKPCB not set", __func__));
1995
1996         /*
1997          * When not using RSS, use connection groups in preference to the
1998          * reservation table when looking up 4-tuples.  When using RSS, just
1999          * use the reservation table, due to the cost of the Toeplitz hash
2000          * in software.
2001          *
2002          * XXXRW: This policy belongs in the pcbgroup code, as in principle
2003          * we could be doing RSS with a non-Toeplitz hash that is affordable
2004          * in software.
2005          */
2006 #if defined(PCBGROUP) && !defined(RSS)
2007         if (in_pcbgroup_enabled(pcbinfo)) {
2008                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2009                     fport);
2010                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2011                     laddr, lport, lookupflags, ifp));
2012         }
2013 #endif
2014         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2015             lookupflags, ifp));
2016 }
2017
2018 struct inpcb *
2019 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2020     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2021     struct ifnet *ifp, struct mbuf *m)
2022 {
2023 #ifdef PCBGROUP
2024         struct inpcbgroup *pcbgroup;
2025 #endif
2026
2027         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2028             ("%s: invalid lookup flags %d", __func__, lookupflags));
2029         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2030             ("%s: LOCKPCB not set", __func__));
2031
2032 #ifdef PCBGROUP
2033         /*
2034          * If we can use a hardware-generated hash to look up the connection
2035          * group, use that connection group to find the inpcb.  Otherwise
2036          * fall back on a software hash -- or the reservation table if we're
2037          * using RSS.
2038          *
2039          * XXXRW: As above, that policy belongs in the pcbgroup code.
2040          */
2041         if (in_pcbgroup_enabled(pcbinfo) &&
2042             !(M_HASHTYPE_TEST(m, M_HASHTYPE_NONE))) {
2043                 pcbgroup = in_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
2044                     m->m_pkthdr.flowid);
2045                 if (pcbgroup != NULL)
2046                         return (in_pcblookup_group(pcbinfo, pcbgroup, faddr,
2047                             fport, laddr, lport, lookupflags, ifp));
2048 #ifndef RSS
2049                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2050                     fport);
2051                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2052                     laddr, lport, lookupflags, ifp));
2053 #endif
2054         }
2055 #endif
2056         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2057             lookupflags, ifp));
2058 }
2059 #endif /* INET */
2060
2061 /*
2062  * Insert PCB onto various hash lists.
2063  */
2064 static int
2065 in_pcbinshash_internal(struct inpcb *inp, int do_pcbgroup_update)
2066 {
2067         struct inpcbhead *pcbhash;
2068         struct inpcbporthead *pcbporthash;
2069         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2070         struct inpcbport *phd;
2071         u_int32_t hashkey_faddr;
2072
2073         INP_WLOCK_ASSERT(inp);
2074         INP_HASH_WLOCK_ASSERT(pcbinfo);
2075
2076         KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2077             ("in_pcbinshash: INP_INHASHLIST"));
2078
2079 #ifdef INET6
2080         if (inp->inp_vflag & INP_IPV6)
2081                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2082         else
2083 #endif
2084         hashkey_faddr = inp->inp_faddr.s_addr;
2085
2086         pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2087                  inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2088
2089         pcbporthash = &pcbinfo->ipi_porthashbase[
2090             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2091
2092         /*
2093          * Go through port list and look for a head for this lport.
2094          */
2095         LIST_FOREACH(phd, pcbporthash, phd_hash) {
2096                 if (phd->phd_port == inp->inp_lport)
2097                         break;
2098         }
2099         /*
2100          * If none exists, malloc one and tack it on.
2101          */
2102         if (phd == NULL) {
2103                 phd = malloc(sizeof(struct inpcbport), M_PCB, M_NOWAIT);
2104                 if (phd == NULL) {
2105                         return (ENOBUFS); /* XXX */
2106                 }
2107                 phd->phd_port = inp->inp_lport;
2108                 LIST_INIT(&phd->phd_pcblist);
2109                 LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2110         }
2111         inp->inp_phd = phd;
2112         LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2113         LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
2114         inp->inp_flags |= INP_INHASHLIST;
2115 #ifdef PCBGROUP
2116         if (do_pcbgroup_update)
2117                 in_pcbgroup_update(inp);
2118 #endif
2119         return (0);
2120 }
2121
2122 /*
2123  * For now, there are two public interfaces to insert an inpcb into the hash
2124  * lists -- one that does update pcbgroups, and one that doesn't.  The latter
2125  * is used only in the TCP syncache, where in_pcbinshash is called before the
2126  * full 4-tuple is set for the inpcb, and we don't want to install in the
2127  * pcbgroup until later.
2128  *
2129  * XXXRW: This seems like a misfeature.  in_pcbinshash should always update
2130  * connection groups, and partially initialised inpcbs should not be exposed
2131  * to either reservation hash tables or pcbgroups.
2132  */
2133 int
2134 in_pcbinshash(struct inpcb *inp)
2135 {
2136
2137         return (in_pcbinshash_internal(inp, 1));
2138 }
2139
2140 int
2141 in_pcbinshash_nopcbgroup(struct inpcb *inp)
2142 {
2143
2144         return (in_pcbinshash_internal(inp, 0));
2145 }
2146
2147 /*
2148  * Move PCB to the proper hash bucket when { faddr, fport } have  been
2149  * changed. NOTE: This does not handle the case of the lport changing (the
2150  * hashed port list would have to be updated as well), so the lport must
2151  * not change after in_pcbinshash() has been called.
2152  */
2153 void
2154 in_pcbrehash_mbuf(struct inpcb *inp, struct mbuf *m)
2155 {
2156         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2157         struct inpcbhead *head;
2158         u_int32_t hashkey_faddr;
2159
2160         INP_WLOCK_ASSERT(inp);
2161         INP_HASH_WLOCK_ASSERT(pcbinfo);
2162
2163         KASSERT(inp->inp_flags & INP_INHASHLIST,
2164             ("in_pcbrehash: !INP_INHASHLIST"));
2165
2166 #ifdef INET6
2167         if (inp->inp_vflag & INP_IPV6)
2168                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2169         else
2170 #endif
2171         hashkey_faddr = inp->inp_faddr.s_addr;
2172
2173         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2174                 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2175
2176         LIST_REMOVE(inp, inp_hash);
2177         LIST_INSERT_HEAD(head, inp, inp_hash);
2178
2179 #ifdef PCBGROUP
2180         if (m != NULL)
2181                 in_pcbgroup_update_mbuf(inp, m);
2182         else
2183                 in_pcbgroup_update(inp);
2184 #endif
2185 }
2186
2187 void
2188 in_pcbrehash(struct inpcb *inp)
2189 {
2190
2191         in_pcbrehash_mbuf(inp, NULL);
2192 }
2193
2194 /*
2195  * Remove PCB from various lists.
2196  */
2197 static void
2198 in_pcbremlists(struct inpcb *inp)
2199 {
2200         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2201
2202 #ifdef INVARIANTS
2203         if (pcbinfo == &V_tcbinfo) {
2204                 INP_INFO_RLOCK_ASSERT(pcbinfo);
2205         } else {
2206                 INP_INFO_WLOCK_ASSERT(pcbinfo);
2207         }
2208 #endif
2209
2210         INP_WLOCK_ASSERT(inp);
2211         INP_LIST_WLOCK_ASSERT(pcbinfo);
2212
2213         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
2214         if (inp->inp_flags & INP_INHASHLIST) {
2215                 struct inpcbport *phd = inp->inp_phd;
2216
2217                 INP_HASH_WLOCK(pcbinfo);
2218                 LIST_REMOVE(inp, inp_hash);
2219                 LIST_REMOVE(inp, inp_portlist);
2220                 if (LIST_FIRST(&phd->phd_pcblist) == NULL) {
2221                         LIST_REMOVE(phd, phd_hash);
2222                         free(phd, M_PCB);
2223                 }
2224                 INP_HASH_WUNLOCK(pcbinfo);
2225                 inp->inp_flags &= ~INP_INHASHLIST;
2226         }
2227         LIST_REMOVE(inp, inp_list);
2228         pcbinfo->ipi_count--;
2229 #ifdef PCBGROUP
2230         in_pcbgroup_remove(inp);
2231 #endif
2232 }
2233
2234 /*
2235  * Check for alternatives when higher level complains
2236  * about service problems.  For now, invalidate cached
2237  * routing information.  If the route was created dynamically
2238  * (by a redirect), time to try a default gateway again.
2239  */
2240 void
2241 in_losing(struct inpcb *inp)
2242 {
2243
2244         RO_RTFREE(&inp->inp_route);
2245         if (inp->inp_route.ro_lle)
2246                 LLE_FREE(inp->inp_route.ro_lle);        /* zeros ro_lle */
2247         return;
2248 }
2249
2250 /*
2251  * A set label operation has occurred at the socket layer, propagate the
2252  * label change into the in_pcb for the socket.
2253  */
2254 void
2255 in_pcbsosetlabel(struct socket *so)
2256 {
2257 #ifdef MAC
2258         struct inpcb *inp;
2259
2260         inp = sotoinpcb(so);
2261         KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2262
2263         INP_WLOCK(inp);
2264         SOCK_LOCK(so);
2265         mac_inpcb_sosetlabel(so, inp);
2266         SOCK_UNLOCK(so);
2267         INP_WUNLOCK(inp);
2268 #endif
2269 }
2270
2271 /*
2272  * ipport_tick runs once per second, determining if random port allocation
2273  * should be continued.  If more than ipport_randomcps ports have been
2274  * allocated in the last second, then we return to sequential port
2275  * allocation. We return to random allocation only once we drop below
2276  * ipport_randomcps for at least ipport_randomtime seconds.
2277  */
2278 static void
2279 ipport_tick(void *xtp)
2280 {
2281         VNET_ITERATOR_DECL(vnet_iter);
2282
2283         VNET_LIST_RLOCK_NOSLEEP();
2284         VNET_FOREACH(vnet_iter) {
2285                 CURVNET_SET(vnet_iter); /* XXX appease INVARIANTS here */
2286                 if (V_ipport_tcpallocs <=
2287                     V_ipport_tcplastcount + V_ipport_randomcps) {
2288                         if (V_ipport_stoprandom > 0)
2289                                 V_ipport_stoprandom--;
2290                 } else
2291                         V_ipport_stoprandom = V_ipport_randomtime;
2292                 V_ipport_tcplastcount = V_ipport_tcpallocs;
2293                 CURVNET_RESTORE();
2294         }
2295         VNET_LIST_RUNLOCK_NOSLEEP();
2296         callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
2297 }
2298
2299 static void
2300 ip_fini(void *xtp)
2301 {
2302
2303         callout_stop(&ipport_tick_callout);
2304 }
2305
2306 /*
2307  * The ipport_callout should start running at about the time we attach the
2308  * inet or inet6 domains.
2309  */
2310 static void
2311 ipport_tick_init(const void *unused __unused)
2312 {
2313
2314         /* Start ipport_tick. */
2315         callout_init(&ipport_tick_callout, 1);
2316         callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
2317         EVENTHANDLER_REGISTER(shutdown_pre_sync, ip_fini, NULL,
2318                 SHUTDOWN_PRI_DEFAULT);
2319 }
2320 SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
2321     ipport_tick_init, NULL);
2322
2323 void
2324 inp_wlock(struct inpcb *inp)
2325 {
2326
2327         INP_WLOCK(inp);
2328 }
2329
2330 void
2331 inp_wunlock(struct inpcb *inp)
2332 {
2333
2334         INP_WUNLOCK(inp);
2335 }
2336
2337 void
2338 inp_rlock(struct inpcb *inp)
2339 {
2340
2341         INP_RLOCK(inp);
2342 }
2343
2344 void
2345 inp_runlock(struct inpcb *inp)
2346 {
2347
2348         INP_RUNLOCK(inp);
2349 }
2350
2351 #ifdef INVARIANTS
2352 void
2353 inp_lock_assert(struct inpcb *inp)
2354 {
2355
2356         INP_WLOCK_ASSERT(inp);
2357 }
2358
2359 void
2360 inp_unlock_assert(struct inpcb *inp)
2361 {
2362
2363         INP_UNLOCK_ASSERT(inp);
2364 }
2365 #endif
2366
2367 void
2368 inp_apply_all(void (*func)(struct inpcb *, void *), void *arg)
2369 {
2370         struct inpcb *inp;
2371
2372         INP_INFO_WLOCK(&V_tcbinfo);
2373         LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) {
2374                 INP_WLOCK(inp);
2375                 func(inp, arg);
2376                 INP_WUNLOCK(inp);
2377         }
2378         INP_INFO_WUNLOCK(&V_tcbinfo);
2379 }
2380
2381 struct socket *
2382 inp_inpcbtosocket(struct inpcb *inp)
2383 {
2384
2385         INP_WLOCK_ASSERT(inp);
2386         return (inp->inp_socket);
2387 }
2388
2389 struct tcpcb *
2390 inp_inpcbtotcpcb(struct inpcb *inp)
2391 {
2392
2393         INP_WLOCK_ASSERT(inp);
2394         return ((struct tcpcb *)inp->inp_ppcb);
2395 }
2396
2397 int
2398 inp_ip_tos_get(const struct inpcb *inp)
2399 {
2400
2401         return (inp->inp_ip_tos);
2402 }
2403
2404 void
2405 inp_ip_tos_set(struct inpcb *inp, int val)
2406 {
2407
2408         inp->inp_ip_tos = val;
2409 }
2410
2411 void
2412 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2413     uint32_t *faddr, uint16_t *fp)
2414 {
2415
2416         INP_LOCK_ASSERT(inp);
2417         *laddr = inp->inp_laddr.s_addr;
2418         *faddr = inp->inp_faddr.s_addr;
2419         *lp = inp->inp_lport;
2420         *fp = inp->inp_fport;
2421 }
2422
2423 struct inpcb *
2424 so_sotoinpcb(struct socket *so)
2425 {
2426
2427         return (sotoinpcb(so));
2428 }
2429
2430 struct tcpcb *
2431 so_sototcpcb(struct socket *so)
2432 {
2433
2434         return (sototcpcb(so));
2435 }
2436
2437 #ifdef DDB
2438 static void
2439 db_print_indent(int indent)
2440 {
2441         int i;
2442
2443         for (i = 0; i < indent; i++)
2444                 db_printf(" ");
2445 }
2446
2447 static void
2448 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
2449 {
2450         char faddr_str[48], laddr_str[48];
2451
2452         db_print_indent(indent);
2453         db_printf("%s at %p\n", name, inc);
2454
2455         indent += 2;
2456
2457 #ifdef INET6
2458         if (inc->inc_flags & INC_ISIPV6) {
2459                 /* IPv6. */
2460                 ip6_sprintf(laddr_str, &inc->inc6_laddr);
2461                 ip6_sprintf(faddr_str, &inc->inc6_faddr);
2462         } else
2463 #endif
2464         {
2465                 /* IPv4. */
2466                 inet_ntoa_r(inc->inc_laddr, laddr_str);
2467                 inet_ntoa_r(inc->inc_faddr, faddr_str);
2468         }
2469         db_print_indent(indent);
2470         db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
2471             ntohs(inc->inc_lport));
2472         db_print_indent(indent);
2473         db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
2474             ntohs(inc->inc_fport));
2475 }
2476
2477 static void
2478 db_print_inpflags(int inp_flags)
2479 {
2480         int comma;
2481
2482         comma = 0;
2483         if (inp_flags & INP_RECVOPTS) {
2484                 db_printf("%sINP_RECVOPTS", comma ? ", " : "");
2485                 comma = 1;
2486         }
2487         if (inp_flags & INP_RECVRETOPTS) {
2488                 db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
2489                 comma = 1;
2490         }
2491         if (inp_flags & INP_RECVDSTADDR) {
2492                 db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
2493                 comma = 1;
2494         }
2495         if (inp_flags & INP_HDRINCL) {
2496                 db_printf("%sINP_HDRINCL", comma ? ", " : "");
2497                 comma = 1;
2498         }
2499         if (inp_flags & INP_HIGHPORT) {
2500                 db_printf("%sINP_HIGHPORT", comma ? ", " : "");
2501                 comma = 1;
2502         }
2503         if (inp_flags & INP_LOWPORT) {
2504                 db_printf("%sINP_LOWPORT", comma ? ", " : "");
2505                 comma = 1;
2506         }
2507         if (inp_flags & INP_ANONPORT) {
2508                 db_printf("%sINP_ANONPORT", comma ? ", " : "");
2509                 comma = 1;
2510         }
2511         if (inp_flags & INP_RECVIF) {
2512                 db_printf("%sINP_RECVIF", comma ? ", " : "");
2513                 comma = 1;
2514         }
2515         if (inp_flags & INP_MTUDISC) {
2516                 db_printf("%sINP_MTUDISC", comma ? ", " : "");
2517                 comma = 1;
2518         }
2519         if (inp_flags & INP_RECVTTL) {
2520                 db_printf("%sINP_RECVTTL", comma ? ", " : "");
2521                 comma = 1;
2522         }
2523         if (inp_flags & INP_DONTFRAG) {
2524                 db_printf("%sINP_DONTFRAG", comma ? ", " : "");
2525                 comma = 1;
2526         }
2527         if (inp_flags & INP_RECVTOS) {
2528                 db_printf("%sINP_RECVTOS", comma ? ", " : "");
2529                 comma = 1;
2530         }
2531         if (inp_flags & IN6P_IPV6_V6ONLY) {
2532                 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
2533                 comma = 1;
2534         }
2535         if (inp_flags & IN6P_PKTINFO) {
2536                 db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
2537                 comma = 1;
2538         }
2539         if (inp_flags & IN6P_HOPLIMIT) {
2540                 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
2541                 comma = 1;
2542         }
2543         if (inp_flags & IN6P_HOPOPTS) {
2544                 db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
2545                 comma = 1;
2546         }
2547         if (inp_flags & IN6P_DSTOPTS) {
2548                 db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
2549                 comma = 1;
2550         }
2551         if (inp_flags & IN6P_RTHDR) {
2552                 db_printf("%sIN6P_RTHDR", comma ? ", " : "");
2553                 comma = 1;
2554         }
2555         if (inp_flags & IN6P_RTHDRDSTOPTS) {
2556                 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
2557                 comma = 1;
2558         }
2559         if (inp_flags & IN6P_TCLASS) {
2560                 db_printf("%sIN6P_TCLASS", comma ? ", " : "");
2561                 comma = 1;
2562         }
2563         if (inp_flags & IN6P_AUTOFLOWLABEL) {
2564                 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
2565                 comma = 1;
2566         }
2567         if (inp_flags & INP_TIMEWAIT) {
2568                 db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
2569                 comma  = 1;
2570         }
2571         if (inp_flags & INP_ONESBCAST) {
2572                 db_printf("%sINP_ONESBCAST", comma ? ", " : "");
2573                 comma  = 1;
2574         }
2575         if (inp_flags & INP_DROPPED) {
2576                 db_printf("%sINP_DROPPED", comma ? ", " : "");
2577                 comma  = 1;
2578         }
2579         if (inp_flags & INP_SOCKREF) {
2580                 db_printf("%sINP_SOCKREF", comma ? ", " : "");
2581                 comma  = 1;
2582         }
2583         if (inp_flags & IN6P_RFC2292) {
2584                 db_printf("%sIN6P_RFC2292", comma ? ", " : "");
2585                 comma = 1;
2586         }
2587         if (inp_flags & IN6P_MTU) {
2588                 db_printf("IN6P_MTU%s", comma ? ", " : "");
2589                 comma = 1;
2590         }
2591 }
2592
2593 static void
2594 db_print_inpvflag(u_char inp_vflag)
2595 {
2596         int comma;
2597
2598         comma = 0;
2599         if (inp_vflag & INP_IPV4) {
2600                 db_printf("%sINP_IPV4", comma ? ", " : "");
2601                 comma  = 1;
2602         }
2603         if (inp_vflag & INP_IPV6) {
2604                 db_printf("%sINP_IPV6", comma ? ", " : "");
2605                 comma  = 1;
2606         }
2607         if (inp_vflag & INP_IPV6PROTO) {
2608                 db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
2609                 comma  = 1;
2610         }
2611 }
2612
2613 static void
2614 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
2615 {
2616
2617         db_print_indent(indent);
2618         db_printf("%s at %p\n", name, inp);
2619
2620         indent += 2;
2621
2622         db_print_indent(indent);
2623         db_printf("inp_flow: 0x%x\n", inp->inp_flow);
2624
2625         db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
2626
2627         db_print_indent(indent);
2628         db_printf("inp_ppcb: %p   inp_pcbinfo: %p   inp_socket: %p\n",
2629             inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
2630
2631         db_print_indent(indent);
2632         db_printf("inp_label: %p   inp_flags: 0x%x (",
2633            inp->inp_label, inp->inp_flags);
2634         db_print_inpflags(inp->inp_flags);
2635         db_printf(")\n");
2636
2637         db_print_indent(indent);
2638         db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
2639             inp->inp_vflag);
2640         db_print_inpvflag(inp->inp_vflag);
2641         db_printf(")\n");
2642
2643         db_print_indent(indent);
2644         db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
2645             inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
2646
2647         db_print_indent(indent);
2648 #ifdef INET6
2649         if (inp->inp_vflag & INP_IPV6) {
2650                 db_printf("in6p_options: %p   in6p_outputopts: %p   "
2651                     "in6p_moptions: %p\n", inp->in6p_options,
2652                     inp->in6p_outputopts, inp->in6p_moptions);
2653                 db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
2654                     "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
2655                     inp->in6p_hops);
2656         } else
2657 #endif
2658         {
2659                 db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
2660                     "inp_ip_moptions: %p\n", inp->inp_ip_tos,
2661                     inp->inp_options, inp->inp_moptions);
2662         }
2663
2664         db_print_indent(indent);
2665         db_printf("inp_phd: %p   inp_gencnt: %ju\n", inp->inp_phd,
2666             (uintmax_t)inp->inp_gencnt);
2667 }
2668
2669 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
2670 {
2671         struct inpcb *inp;
2672
2673         if (!have_addr) {
2674                 db_printf("usage: show inpcb <addr>\n");
2675                 return;
2676         }
2677         inp = (struct inpcb *)addr;
2678
2679         db_print_inpcb(inp, "inpcb", 0);
2680 }
2681 #endif /* DDB */
2682
2683 #ifdef RATELIMIT
2684 /*
2685  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
2686  * if any.
2687  */
2688 int
2689 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
2690 {
2691         union if_snd_tag_modify_params params = {
2692                 .rate_limit.max_rate = max_pacing_rate,
2693         };
2694         struct m_snd_tag *mst;
2695         struct ifnet *ifp;
2696         int error;
2697
2698         mst = inp->inp_snd_tag;
2699         if (mst == NULL)
2700                 return (EINVAL);
2701
2702         ifp = mst->ifp;
2703         if (ifp == NULL)
2704                 return (EINVAL);
2705
2706         if (ifp->if_snd_tag_modify == NULL) {
2707                 error = EOPNOTSUPP;
2708         } else {
2709                 error = ifp->if_snd_tag_modify(mst, &params);
2710         }
2711         return (error);
2712 }
2713
2714 /*
2715  * Query existing TX rate limit based on the existing
2716  * "inp->inp_snd_tag", if any.
2717  */
2718 int
2719 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
2720 {
2721         union if_snd_tag_query_params params = { };
2722         struct m_snd_tag *mst;
2723         struct ifnet *ifp;
2724         int error;
2725
2726         mst = inp->inp_snd_tag;
2727         if (mst == NULL)
2728                 return (EINVAL);
2729
2730         ifp = mst->ifp;
2731         if (ifp == NULL)
2732                 return (EINVAL);
2733
2734         if (ifp->if_snd_tag_query == NULL) {
2735                 error = EOPNOTSUPP;
2736         } else {
2737                 error = ifp->if_snd_tag_query(mst, &params);
2738                 if (error == 0 &&  p_max_pacing_rate != NULL)
2739                         *p_max_pacing_rate = params.rate_limit.max_rate;
2740         }
2741         return (error);
2742 }
2743
2744 /*
2745  * Allocate a new TX rate limit send tag from the network interface
2746  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
2747  */
2748 int
2749 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
2750     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate)
2751 {
2752         union if_snd_tag_alloc_params params = {
2753                 .rate_limit.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT,
2754                 .rate_limit.hdr.flowid = flowid,
2755                 .rate_limit.hdr.flowtype = flowtype,
2756                 .rate_limit.max_rate = max_pacing_rate,
2757         };
2758         int error;
2759
2760         INP_WLOCK_ASSERT(inp);
2761
2762         if (inp->inp_snd_tag != NULL)
2763                 return (EINVAL);
2764
2765         if (ifp->if_snd_tag_alloc == NULL) {
2766                 error = EOPNOTSUPP;
2767         } else {
2768                 error = ifp->if_snd_tag_alloc(ifp, &params, &inp->inp_snd_tag);
2769
2770                 /*
2771                  * At success increment the refcount on
2772                  * the send tag's network interface:
2773                  */
2774                 if (error == 0)
2775                         if_ref(inp->inp_snd_tag->ifp);
2776         }
2777         return (error);
2778 }
2779
2780 /*
2781  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
2782  * if any:
2783  */
2784 void
2785 in_pcbdetach_txrtlmt(struct inpcb *inp)
2786 {
2787         struct m_snd_tag *mst;
2788         struct ifnet *ifp;
2789
2790         INP_WLOCK_ASSERT(inp);
2791
2792         mst = inp->inp_snd_tag;
2793         inp->inp_snd_tag = NULL;
2794
2795         if (mst == NULL)
2796                 return;
2797
2798         ifp = mst->ifp;
2799         if (ifp == NULL)
2800                 return;
2801
2802         /*
2803          * If the device was detached while we still had reference(s)
2804          * on the ifp, we assume if_snd_tag_free() was replaced with
2805          * stubs.
2806          */
2807         ifp->if_snd_tag_free(mst);
2808
2809         /* release reference count on network interface */
2810         if_rele(ifp);
2811 }
2812
2813 /*
2814  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
2815  * is set in the fast path and will attach/detach/modify the TX rate
2816  * limit send tag based on the socket's so_max_pacing_rate value.
2817  */
2818 void
2819 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
2820 {
2821         struct socket *socket;
2822         uint32_t max_pacing_rate;
2823         bool did_upgrade;
2824         int error;
2825
2826         if (inp == NULL)
2827                 return;
2828
2829         socket = inp->inp_socket;
2830         if (socket == NULL)
2831                 return;
2832
2833         if (!INP_WLOCKED(inp)) {
2834                 /*
2835                  * NOTE: If the write locking fails, we need to bail
2836                  * out and use the non-ratelimited ring for the
2837                  * transmit until there is a new chance to get the
2838                  * write lock.
2839                  */
2840                 if (!INP_TRY_UPGRADE(inp))
2841                         return;
2842                 did_upgrade = 1;
2843         } else {
2844                 did_upgrade = 0;
2845         }
2846
2847         /*
2848          * NOTE: The so_max_pacing_rate value is read unlocked,
2849          * because atomic updates are not required since the variable
2850          * is checked at every mbuf we send. It is assumed that the
2851          * variable read itself will be atomic.
2852          */
2853         max_pacing_rate = socket->so_max_pacing_rate;
2854
2855         /*
2856          * NOTE: When attaching to a network interface a reference is
2857          * made to ensure the network interface doesn't go away until
2858          * all ratelimit connections are gone. The network interface
2859          * pointers compared below represent valid network interfaces,
2860          * except when comparing towards NULL.
2861          */
2862         if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
2863                 error = 0;
2864         } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
2865                 if (inp->inp_snd_tag != NULL)
2866                         in_pcbdetach_txrtlmt(inp);
2867                 error = 0;
2868         } else if (inp->inp_snd_tag == NULL) {
2869                 /*
2870                  * In order to utilize packet pacing with RSS, we need
2871                  * to wait until there is a valid RSS hash before we
2872                  * can proceed:
2873                  */
2874                 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
2875                         error = EAGAIN;
2876                 } else {
2877                         error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
2878                             mb->m_pkthdr.flowid, max_pacing_rate);
2879                 }
2880         } else {
2881                 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
2882         }
2883         if (error == 0 || error == EOPNOTSUPP)
2884                 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
2885         if (did_upgrade)
2886                 INP_DOWNGRADE(inp);
2887 }
2888
2889 /*
2890  * Track route changes for TX rate limiting.
2891  */
2892 void
2893 in_pcboutput_eagain(struct inpcb *inp)
2894 {
2895         struct socket *socket;
2896         bool did_upgrade;
2897
2898         if (inp == NULL)
2899                 return;
2900
2901         socket = inp->inp_socket;
2902         if (socket == NULL)
2903                 return;
2904
2905         if (inp->inp_snd_tag == NULL)
2906                 return;
2907
2908         if (!INP_WLOCKED(inp)) {
2909                 /*
2910                  * NOTE: If the write locking fails, we need to bail
2911                  * out and use the non-ratelimited ring for the
2912                  * transmit until there is a new chance to get the
2913                  * write lock.
2914                  */
2915                 if (!INP_TRY_UPGRADE(inp))
2916                         return;
2917                 did_upgrade = 1;
2918         } else {
2919                 did_upgrade = 0;
2920         }
2921
2922         /* detach rate limiting */
2923         in_pcbdetach_txrtlmt(inp);
2924
2925         /* make sure new mbuf send tag allocation is made */
2926         inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
2927
2928         if (did_upgrade)
2929                 INP_DOWNGRADE(inp);
2930 }
2931 #endif /* RATELIMIT */