sys/netinet/in_pcb.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 1982, 1986, 1991, 1993, 1995
   5  *      The Regents of the University of California.
   6  * Copyright (c) 2007-2009 Robert N. M. Watson
   7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
   8  * All rights reserved.
   9  *
  10  * Portions of this software were developed by Robert N. M. Watson under
  11  * contract to Juniper Networks, Inc.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  * 3. Neither the name of the University nor the names of its contributors
  22  *    may be used to endorse or promote products derived from this software
  23  *    without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  *
  37  *      @(#)in_pcb.c    8.4 (Berkeley) 5/24/95
  38  */
  39
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include "opt_ddb.h"
  44 #include "opt_ipsec.h"
  45 #include "opt_inet.h"
  46 #include "opt_inet6.h"
  47 #include "opt_ratelimit.h"
  48 #include "opt_pcbgroup.h"
  49 #include "opt_rss.h"
  50
  51 #include <sys/param.h>
  52 #include <sys/systm.h>
  53 #include <sys/lock.h>
  54 #include <sys/malloc.h>
  55 #include <sys/mbuf.h>
  56 #include <sys/callout.h>
  57 #include <sys/eventhandler.h>
  58 #include <sys/domain.h>
  59 #include <sys/protosw.h>
  60 #include <sys/rmlock.h>
  61 #include <sys/smp.h>
  62 #include <sys/socket.h>
  63 #include <sys/socketvar.h>
  64 #include <sys/sockio.h>
  65 #include <sys/priv.h>
  66 #include <sys/proc.h>
  67 #include <sys/refcount.h>
  68 #include <sys/jail.h>
  69 #include <sys/kernel.h>
  70 #include <sys/sysctl.h>
  71
  72 #ifdef DDB
  73 #include <ddb/ddb.h>
  74 #endif
  75
  76 #include <vm/uma.h>
  77
  78 #include <net/if.h>
  79 #include <net/if_var.h>
  80 #include <net/if_types.h>
  81 #include <net/if_llatbl.h>
  82 #include <net/route.h>
  83 #include <net/rss_config.h>
  84 #include <net/vnet.h>
  85
  86 #if defined(INET) || defined(INET6)
  87 #include <netinet/in.h>
  88 #include <netinet/in_pcb.h>
  89 #ifdef INET
  90 #include <netinet/in_var.h>
  91 #endif
  92 #include <netinet/ip_var.h>
  93 #include <netinet/tcp_var.h>
  94 #ifdef TCPHPTS
  95 #include <netinet/tcp_hpts.h>
  96 #endif
  97 #include <netinet/udp.h>
  98 #include <netinet/udp_var.h>
  99 #ifdef INET6
 100 #include <netinet/ip6.h>
 101 #include <netinet6/in6_pcb.h>
 102 #include <netinet6/in6_var.h>
 103 #include <netinet6/ip6_var.h>
 104 #endif /* INET6 */
 105 #endif
 106
 107 #include <netipsec/ipsec_support.h>
 108
 109 #include <security/mac/mac_framework.h>
 110
 111 #define INPCBLBGROUP_SIZMIN     8
 112 #define INPCBLBGROUP_SIZMAX     256
 113
 114 static struct callout   ipport_tick_callout;
 115
 116 /*
 117  * These configure the range of local port addresses assigned to
 118  * "unspecified" outgoing connections/packets/whatever.
 119  */
 120 VNET_DEFINE(int, ipport_lowfirstauto) = IPPORT_RESERVED - 1;    /* 1023 */
 121 VNET_DEFINE(int, ipport_lowlastauto) = IPPORT_RESERVEDSTART;    /* 600 */
 122 VNET_DEFINE(int, ipport_firstauto) = IPPORT_EPHEMERALFIRST;     /* 10000 */
 123 VNET_DEFINE(int, ipport_lastauto) = IPPORT_EPHEMERALLAST;       /* 65535 */
 124 VNET_DEFINE(int, ipport_hifirstauto) = IPPORT_HIFIRSTAUTO;      /* 49152 */
 125 VNET_DEFINE(int, ipport_hilastauto) = IPPORT_HILASTAUTO;        /* 65535 */
 126
 127 /*
 128  * Reserved ports accessible only to root. There are significant
 129  * security considerations that must be accounted for when changing these,
 130  * but the security benefits can be great. Please be careful.
 131  */
 132 VNET_DEFINE(int, ipport_reservedhigh) = IPPORT_RESERVED - 1;    /* 1023 */
 133 VNET_DEFINE(int, ipport_reservedlow);
 134
 135 /* Variables dealing with random ephemeral port allocation. */
 136 VNET_DEFINE(int, ipport_randomized) = 1;        /* user controlled via sysctl */
 137 VNET_DEFINE(int, ipport_randomcps) = 10;        /* user controlled via sysctl */
 138 VNET_DEFINE(int, ipport_randomtime) = 45;       /* user controlled via sysctl */
 139 VNET_DEFINE(int, ipport_stoprandom);            /* toggled by ipport_tick */
 140 VNET_DEFINE(int, ipport_tcpallocs);
 141 VNET_DEFINE_STATIC(int, ipport_tcplastcount);
 142
 143 #define V_ipport_tcplastcount           VNET(ipport_tcplastcount)
 144
 145 static void     in_pcbremlists(struct inpcb *inp);
 146 #ifdef INET
 147 static struct inpcb     *in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
 148                             struct in_addr faddr, u_int fport_arg,
 149                             struct in_addr laddr, u_int lport_arg,
 150                             int lookupflags, struct ifnet *ifp);
 151
 152 #define RANGECHK(var, min, max) \
 153         if ((var) < (min)) { (var) = (min); } \
 154         else if ((var) > (max)) { (var) = (max); }
 155
 156 static int
 157 sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
 158 {
 159         int error;
 160
 161         error = sysctl_handle_int(oidp, arg1, arg2, req);
 162         if (error == 0) {
 163                 RANGECHK(V_ipport_lowfirstauto, 1, IPPORT_RESERVED - 1);
 164                 RANGECHK(V_ipport_lowlastauto, 1, IPPORT_RESERVED - 1);
 165                 RANGECHK(V_ipport_firstauto, IPPORT_RESERVED, IPPORT_MAX);
 166                 RANGECHK(V_ipport_lastauto, IPPORT_RESERVED, IPPORT_MAX);
 167                 RANGECHK(V_ipport_hifirstauto, IPPORT_RESERVED, IPPORT_MAX);
 168                 RANGECHK(V_ipport_hilastauto, IPPORT_RESERVED, IPPORT_MAX);
 169         }
 170         return (error);
 171 }
 172
 173 #undef RANGECHK
 174
 175 static SYSCTL_NODE(_net_inet_ip, IPPROTO_IP, portrange,
 176     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 177     "IP Ports");
 178
 179 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
 180     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 181     &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check, "I",
 182     "");
 183 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
 184     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 185     &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check, "I",
 186     "");
 187 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
 188     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 189     &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check, "I",
 190     "");
 191 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, last,
 192     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 193     &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check, "I",
 194     "");
 195 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
 196     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 197     &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check, "I",
 198     "");
 199 SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
 200     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 201     &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check, "I",
 202     "");
 203 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
 204         CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
 205         &VNET_NAME(ipport_reservedhigh), 0, "");
 206 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
 207         CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0, "");
 208 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
 209         CTLFLAG_VNET | CTLFLAG_RW,
 210         &VNET_NAME(ipport_randomized), 0, "Enable random port allocation");
 211 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
 212         CTLFLAG_VNET | CTLFLAG_RW,
 213         &VNET_NAME(ipport_randomcps), 0, "Maximum number of random port "
 214         "allocations before switching to a sequental one");
 215 SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
 216         CTLFLAG_VNET | CTLFLAG_RW,
 217         &VNET_NAME(ipport_randomtime), 0,
 218         "Minimum time to keep sequental port "
 219         "allocation before switching to a random one");
 220
 221 #ifdef RATELIMIT
 222 counter_u64_t rate_limit_active;
 223 counter_u64_t rate_limit_alloc_fail;
 224 counter_u64_t rate_limit_set_ok;
 225
 226 static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 227     "IP Rate Limiting");
 228 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, active, CTLFLAG_RD,
 229     &rate_limit_active, "Active rate limited connections");
 230 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, alloc_fail, CTLFLAG_RD,
 231    &rate_limit_alloc_fail, "Rate limited connection failures");
 232 SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, CTLFLAG_RD,
 233    &rate_limit_set_ok, "Rate limited setting succeeded");
 234 #endif /* RATELIMIT */
 235
 236 #endif /* INET */
 237
 238 /*
 239  * in_pcb.c: manage the Protocol Control Blocks.
 240  *
 241  * NOTE: It is assumed that most of these functions will be called with
 242  * the pcbinfo lock held, and often, the inpcb lock held, as these utility
 243  * functions often modify hash chains or addresses in pcbs.
 244  */
 245
 246 static struct inpcblbgroup *
 247 in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag,
 248     uint16_t port, const union in_dependaddr *addr, int size)
 249 {
 250         struct inpcblbgroup *grp;
 251         size_t bytes;
 252
 253         bytes = __offsetof(struct inpcblbgroup, il_inp[size]);
 254         grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
 255         if (!grp)
 256                 return (NULL);
 257         grp->il_vflag = vflag;
 258         grp->il_lport = port;
 259         grp->il_dependladdr = *addr;
 260         grp->il_inpsiz = size;
 261         CK_LIST_INSERT_HEAD(hdr, grp, il_list);
 262         return (grp);
 263 }
 264
 265 static void
 266 in_pcblbgroup_free_deferred(epoch_context_t ctx)
 267 {
 268         struct inpcblbgroup *grp;
 269
 270         grp = __containerof(ctx, struct inpcblbgroup, il_epoch_ctx);
 271         free(grp, M_PCB);
 272 }
 273
 274 static void
 275 in_pcblbgroup_free(struct inpcblbgroup *grp)
 276 {
 277
 278         CK_LIST_REMOVE(grp, il_list);
 279         NET_EPOCH_CALL(in_pcblbgroup_free_deferred, &grp->il_epoch_ctx);
 280 }
 281
 282 static struct inpcblbgroup *
 283 in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
 284     struct inpcblbgroup *old_grp, int size)
 285 {
 286         struct inpcblbgroup *grp;
 287         int i;
 288
 289         grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag,
 290             old_grp->il_lport, &old_grp->il_dependladdr, size);
 291         if (grp == NULL)
 292                 return (NULL);
 293
 294         KASSERT(old_grp->il_inpcnt < grp->il_inpsiz,
 295             ("invalid new local group size %d and old local group count %d",
 296              grp->il_inpsiz, old_grp->il_inpcnt));
 297
 298         for (i = 0; i < old_grp->il_inpcnt; ++i)
 299                 grp->il_inp[i] = old_grp->il_inp[i];
 300         grp->il_inpcnt = old_grp->il_inpcnt;
 301         in_pcblbgroup_free(old_grp);
 302         return (grp);
 303 }
 304
 305 /*
 306  * PCB at index 'i' is removed from the group. Pull up the ones below il_inp[i]
 307  * and shrink group if possible.
 308  */
 309 static void
 310 in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, struct inpcblbgroup **grpp,
 311     int i)
 312 {
 313         struct inpcblbgroup *grp, *new_grp;
 314
 315         grp = *grpp;
 316         for (; i + 1 < grp->il_inpcnt; ++i)
 317                 grp->il_inp[i] = grp->il_inp[i + 1];
 318         grp->il_inpcnt--;
 319
 320         if (grp->il_inpsiz > INPCBLBGROUP_SIZMIN &&
 321             grp->il_inpcnt <= grp->il_inpsiz / 4) {
 322                 /* Shrink this group. */
 323                 new_grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz / 2);
 324                 if (new_grp != NULL)
 325                         *grpp = new_grp;
 326         }
 327 }
 328
 329 /*
 330  * Add PCB to load balance group for SO_REUSEPORT_LB option.
 331  */
 332 static int
 333 in_pcbinslbgrouphash(struct inpcb *inp)
 334 {
 335         const static struct timeval interval = { 60, 0 };
 336         static struct timeval lastprint;
 337         struct inpcbinfo *pcbinfo;
 338         struct inpcblbgrouphead *hdr;
 339         struct inpcblbgroup *grp;
 340         uint32_t idx;
 341
 342         pcbinfo = inp->inp_pcbinfo;
 343
 344         INP_WLOCK_ASSERT(inp);
 345         INP_HASH_WLOCK_ASSERT(pcbinfo);
 346
 347         /*
 348          * Don't allow jailed socket to join local group.
 349          */
 350         if (inp->inp_socket != NULL && jailed(inp->inp_socket->so_cred))
 351                 return (0);
 352
 353 #ifdef INET6
 354         /*
 355          * Don't allow IPv4 mapped INET6 wild socket.
 356          */
 357         if ((inp->inp_vflag & INP_IPV4) &&
 358             inp->inp_laddr.s_addr == INADDR_ANY &&
 359             INP_CHECK_SOCKAF(inp->inp_socket, AF_INET6)) {
 360                 return (0);
 361         }
 362 #endif
 363
 364         idx = INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask);
 365         hdr = &pcbinfo->ipi_lbgrouphashbase[idx];
 366         CK_LIST_FOREACH(grp, hdr, il_list) {
 367                 if (grp->il_vflag == inp->inp_vflag &&
 368                     grp->il_lport == inp->inp_lport &&
 369                     memcmp(&grp->il_dependladdr,
 370                     &inp->inp_inc.inc_ie.ie_dependladdr,
 371                     sizeof(grp->il_dependladdr)) == 0)
 372                         break;
 373         }
 374         if (grp == NULL) {
 375                 /* Create new load balance group. */
 376                 grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag,
 377                     inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
 378                     INPCBLBGROUP_SIZMIN);
 379                 if (grp == NULL)
 380                         return (ENOBUFS);
 381         } else if (grp->il_inpcnt == grp->il_inpsiz) {
 382                 if (grp->il_inpsiz >= INPCBLBGROUP_SIZMAX) {
 383                         if (ratecheck(&lastprint, &interval))
 384                                 printf("lb group port %d, limit reached\n",
 385                                     ntohs(grp->il_lport));
 386                         return (0);
 387                 }
 388
 389                 /* Expand this local group. */
 390                 grp = in_pcblbgroup_resize(hdr, grp, grp->il_inpsiz * 2);
 391                 if (grp == NULL)
 392                         return (ENOBUFS);
 393         }
 394
 395         KASSERT(grp->il_inpcnt < grp->il_inpsiz,
 396             ("invalid local group size %d and count %d", grp->il_inpsiz,
 397             grp->il_inpcnt));
 398
 399         grp->il_inp[grp->il_inpcnt] = inp;
 400         grp->il_inpcnt++;
 401         return (0);
 402 }
 403
 404 /*
 405  * Remove PCB from load balance group.
 406  */
 407 static void
 408 in_pcbremlbgrouphash(struct inpcb *inp)
 409 {
 410         struct inpcbinfo *pcbinfo;
 411         struct inpcblbgrouphead *hdr;
 412         struct inpcblbgroup *grp;
 413         int i;
 414
 415         pcbinfo = inp->inp_pcbinfo;
 416
 417         INP_WLOCK_ASSERT(inp);
 418         INP_HASH_WLOCK_ASSERT(pcbinfo);
 419
 420         hdr = &pcbinfo->ipi_lbgrouphashbase[
 421             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
 422         CK_LIST_FOREACH(grp, hdr, il_list) {
 423                 for (i = 0; i < grp->il_inpcnt; ++i) {
 424                         if (grp->il_inp[i] != inp)
 425                                 continue;
 426
 427                         if (grp->il_inpcnt == 1) {
 428                                 /* We are the last, free this local group. */
 429                                 in_pcblbgroup_free(grp);
 430                         } else {
 431                                 /* Pull up inpcbs, shrink group if possible. */
 432                                 in_pcblbgroup_reorder(hdr, &grp, i);
 433                         }
 434                         return;
 435                 }
 436         }
 437 }
 438
 439 /*
 440  * Different protocols initialize their inpcbs differently - giving
 441  * different name to the lock.  But they all are disposed the same.
 442  */
 443 static void
 444 inpcb_fini(void *mem, int size)
 445 {
 446         struct inpcb *inp = mem;
 447
 448         INP_LOCK_DESTROY(inp);
 449 }
 450
 451 /*
 452  * Initialize an inpcbinfo -- we should be able to reduce the number of
 453  * arguments in time.
 454  */
 455 void
 456 in_pcbinfo_init(struct inpcbinfo *pcbinfo, const char *name,
 457     struct inpcbhead *listhead, int hash_nelements, int porthash_nelements,
 458     char *inpcbzone_name, uma_init inpcbzone_init, u_int hashfields)
 459 {
 460
 461         porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1);
 462
 463         INP_INFO_LOCK_INIT(pcbinfo, name);
 464         INP_HASH_LOCK_INIT(pcbinfo, "pcbinfohash");     /* XXXRW: argument? */
 465         INP_LIST_LOCK_INIT(pcbinfo, "pcbinfolist");
 466 #ifdef VIMAGE
 467         pcbinfo->ipi_vnet = curvnet;
 468 #endif
 469         pcbinfo->ipi_listhead = listhead;
 470         CK_LIST_INIT(pcbinfo->ipi_listhead);
 471         pcbinfo->ipi_count = 0;
 472         pcbinfo->ipi_hashbase = hashinit(hash_nelements, M_PCB,
 473             &pcbinfo->ipi_hashmask);
 474         pcbinfo->ipi_porthashbase = hashinit(porthash_nelements, M_PCB,
 475             &pcbinfo->ipi_porthashmask);
 476         pcbinfo->ipi_lbgrouphashbase = hashinit(porthash_nelements, M_PCB,
 477             &pcbinfo->ipi_lbgrouphashmask);
 478 #ifdef PCBGROUP
 479         in_pcbgroup_init(pcbinfo, hashfields, hash_nelements);
 480 #endif
 481         pcbinfo->ipi_zone = uma_zcreate(inpcbzone_name, sizeof(struct inpcb),
 482             NULL, NULL, inpcbzone_init, inpcb_fini, UMA_ALIGN_PTR, 0);
 483         uma_zone_set_max(pcbinfo->ipi_zone, maxsockets);
 484         uma_zone_set_warning(pcbinfo->ipi_zone,
 485             "kern.ipc.maxsockets limit reached");
 486 }
 487
 488 /*
 489  * Destroy an inpcbinfo.
 490  */
 491 void
 492 in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
 493 {
 494
 495         KASSERT(pcbinfo->ipi_count == 0,
 496             ("%s: ipi_count = %u", __func__, pcbinfo->ipi_count));
 497
 498         hashdestroy(pcbinfo->ipi_hashbase, M_PCB, pcbinfo->ipi_hashmask);
 499         hashdestroy(pcbinfo->ipi_porthashbase, M_PCB,
 500             pcbinfo->ipi_porthashmask);
 501         hashdestroy(pcbinfo->ipi_lbgrouphashbase, M_PCB,
 502             pcbinfo->ipi_lbgrouphashmask);
 503 #ifdef PCBGROUP
 504         in_pcbgroup_destroy(pcbinfo);
 505 #endif
 506         uma_zdestroy(pcbinfo->ipi_zone);
 507         INP_LIST_LOCK_DESTROY(pcbinfo);
 508         INP_HASH_LOCK_DESTROY(pcbinfo);
 509         INP_INFO_LOCK_DESTROY(pcbinfo);
 510 }
 511
 512 /*
 513  * Allocate a PCB and associate it with the socket.
 514  * On success return with the PCB locked.
 515  */
 516 int
 517 in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
 518 {
 519         struct inpcb *inp;
 520         int error;
 521
 522         error = 0;
 523         inp = uma_zalloc(pcbinfo->ipi_zone, M_NOWAIT);
 524         if (inp == NULL)
 525                 return (ENOBUFS);
 526         bzero(&inp->inp_start_zero, inp_zero_size);
 527 #ifdef NUMA
 528         inp->inp_numa_domain = M_NODOM;
 529 #endif
 530         inp->inp_pcbinfo = pcbinfo;
 531         inp->inp_socket = so;
 532         inp->inp_cred = crhold(so->so_cred);
 533         inp->inp_inc.inc_fibnum = so->so_fibnum;
 534 #ifdef MAC
 535         error = mac_inpcb_init(inp, M_NOWAIT);
 536         if (error != 0)
 537                 goto out;
 538         mac_inpcb_create(so, inp);
 539 #endif
 540 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
 541         error = ipsec_init_pcbpolicy(inp);
 542         if (error != 0) {
 543 #ifdef MAC
 544                 mac_inpcb_destroy(inp);
 545 #endif
 546                 goto out;
 547         }
 548 #endif /*IPSEC*/
 549 #ifdef INET6
 550         if (INP_SOCKAF(so) == AF_INET6) {
 551                 inp->inp_vflag |= INP_IPV6PROTO;
 552                 if (V_ip6_v6only)
 553                         inp->inp_flags |= IN6P_IPV6_V6ONLY;
 554         }
 555 #endif
 556         INP_WLOCK(inp);
 557         INP_LIST_WLOCK(pcbinfo);
 558         CK_LIST_INSERT_HEAD(pcbinfo->ipi_listhead, inp, inp_list);
 559         pcbinfo->ipi_count++;
 560         so->so_pcb = (caddr_t)inp;
 561 #ifdef INET6
 562         if (V_ip6_auto_flowlabel)
 563                 inp->inp_flags |= IN6P_AUTOFLOWLABEL;
 564 #endif
 565         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 566         refcount_init(&inp->inp_refcount, 1);   /* Reference from inpcbinfo */
 567
 568         /*
 569          * Routes in inpcb's can cache L2 as well; they are guaranteed
 570          * to be cleaned up.
 571          */
 572         inp->inp_route.ro_flags = RT_LLE_CACHE;
 573         INP_LIST_WUNLOCK(pcbinfo);
 574 #if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
 575 out:
 576         if (error != 0) {
 577                 crfree(inp->inp_cred);
 578                 uma_zfree(pcbinfo->ipi_zone, inp);
 579         }
 580 #endif
 581         return (error);
 582 }
 583
 584 #ifdef INET
 585 int
 586 in_pcbbind(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 587 {
 588         int anonport, error;
 589
 590         INP_WLOCK_ASSERT(inp);
 591         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 592
 593         if (inp->inp_lport != 0 || inp->inp_laddr.s_addr != INADDR_ANY)
 594                 return (EINVAL);
 595         anonport = nam == NULL || ((struct sockaddr_in *)nam)->sin_port == 0;
 596         error = in_pcbbind_setup(inp, nam, &inp->inp_laddr.s_addr,
 597             &inp->inp_lport, cred);
 598         if (error)
 599                 return (error);
 600         if (in_pcbinshash(inp) != 0) {
 601                 inp->inp_laddr.s_addr = INADDR_ANY;
 602                 inp->inp_lport = 0;
 603                 return (EAGAIN);
 604         }
 605         if (anonport)
 606                 inp->inp_flags |= INP_ANONPORT;
 607         return (0);
 608 }
 609 #endif
 610
 611 /*
 612  * Select a local port (number) to use.
 613  */
 614 #if defined(INET) || defined(INET6)
 615 int
 616 in_pcb_lport(struct inpcb *inp, struct in_addr *laddrp, u_short *lportp,
 617     struct ucred *cred, int lookupflags)
 618 {
 619         struct inpcbinfo *pcbinfo;
 620         struct inpcb *tmpinp;
 621         unsigned short *lastport;
 622         int count, dorandom, error;
 623         u_short aux, first, last, lport;
 624 #ifdef INET
 625         struct in_addr laddr;
 626 #endif
 627
 628         pcbinfo = inp->inp_pcbinfo;
 629
 630         /*
 631          * Because no actual state changes occur here, a global write lock on
 632          * the pcbinfo isn't required.
 633          */
 634         INP_LOCK_ASSERT(inp);
 635         INP_HASH_LOCK_ASSERT(pcbinfo);
 636
 637         if (inp->inp_flags & INP_HIGHPORT) {
 638                 first = V_ipport_hifirstauto;   /* sysctl */
 639                 last  = V_ipport_hilastauto;
 640                 lastport = &pcbinfo->ipi_lasthi;
 641         } else if (inp->inp_flags & INP_LOWPORT) {
 642                 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
 643                 if (error)
 644                         return (error);
 645                 first = V_ipport_lowfirstauto;  /* 1023 */
 646                 last  = V_ipport_lowlastauto;   /* 600 */
 647                 lastport = &pcbinfo->ipi_lastlow;
 648         } else {
 649                 first = V_ipport_firstauto;     /* sysctl */
 650                 last  = V_ipport_lastauto;
 651                 lastport = &pcbinfo->ipi_lastport;
 652         }
 653         /*
 654          * For UDP(-Lite), use random port allocation as long as the user
 655          * allows it.  For TCP (and as of yet unknown) connections,
 656          * use random port allocation only if the user allows it AND
 657          * ipport_tick() allows it.
 658          */
 659         if (V_ipport_randomized &&
 660                 (!V_ipport_stoprandom || pcbinfo == &V_udbinfo ||
 661                 pcbinfo == &V_ulitecbinfo))
 662                 dorandom = 1;
 663         else
 664                 dorandom = 0;
 665         /*
 666          * It makes no sense to do random port allocation if
 667          * we have the only port available.
 668          */
 669         if (first == last)
 670                 dorandom = 0;
 671         /* Make sure to not include UDP(-Lite) packets in the count. */
 672         if (pcbinfo != &V_udbinfo || pcbinfo != &V_ulitecbinfo)
 673                 V_ipport_tcpallocs++;
 674         /*
 675          * Instead of having two loops further down counting up or down
 676          * make sure that first is always <= last and go with only one
 677          * code path implementing all logic.
 678          */
 679         if (first > last) {
 680                 aux = first;
 681                 first = last;
 682                 last = aux;
 683         }
 684
 685 #ifdef INET
 686         /* Make the compiler happy. */
 687         laddr.s_addr = 0;
 688         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4) {
 689                 KASSERT(laddrp != NULL, ("%s: laddrp NULL for v4 inp %p",
 690                     __func__, inp));
 691                 laddr = *laddrp;
 692         }
 693 #endif
 694         tmpinp = NULL;  /* Make compiler happy. */
 695         lport = *lportp;
 696
 697         if (dorandom)
 698                 *lastport = first + (arc4random() % (last - first));
 699
 700         count = last - first;
 701
 702         do {
 703                 if (count-- < 0)        /* completely used? */
 704                         return (EADDRNOTAVAIL);
 705                 ++*lastport;
 706                 if (*lastport < first || *lastport > last)
 707                         *lastport = first;
 708                 lport = htons(*lastport);
 709
 710 #ifdef INET6
 711                 if ((inp->inp_vflag & INP_IPV6) != 0)
 712                         tmpinp = in6_pcblookup_local(pcbinfo,
 713                             &inp->in6p_laddr, lport, lookupflags, cred);
 714 #endif
 715 #if defined(INET) && defined(INET6)
 716                 else
 717 #endif
 718 #ifdef INET
 719                         tmpinp = in_pcblookup_local(pcbinfo, laddr,
 720                             lport, lookupflags, cred);
 721 #endif
 722         } while (tmpinp != NULL);
 723
 724 #ifdef INET
 725         if ((inp->inp_vflag & (INP_IPV4|INP_IPV6)) == INP_IPV4)
 726                 laddrp->s_addr = laddr.s_addr;
 727 #endif
 728         *lportp = lport;
 729
 730         return (0);
 731 }
 732
 733 /*
 734  * Return cached socket options.
 735  */
 736 int
 737 inp_so_options(const struct inpcb *inp)
 738 {
 739         int so_options;
 740
 741         so_options = 0;
 742
 743         if ((inp->inp_flags2 & INP_REUSEPORT_LB) != 0)
 744                 so_options |= SO_REUSEPORT_LB;
 745         if ((inp->inp_flags2 & INP_REUSEPORT) != 0)
 746                 so_options |= SO_REUSEPORT;
 747         if ((inp->inp_flags2 & INP_REUSEADDR) != 0)
 748                 so_options |= SO_REUSEADDR;
 749         return (so_options);
 750 }
 751 #endif /* INET || INET6 */
 752
 753 /*
 754  * Check if a new BINDMULTI socket is allowed to be created.
 755  *
 756  * ni points to the new inp.
 757  * oi points to the exisitng inp.
 758  *
 759  * This checks whether the existing inp also has BINDMULTI and
 760  * whether the credentials match.
 761  */
 762 int
 763 in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
 764 {
 765         /* Check permissions match */
 766         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 767             (ni->inp_cred->cr_uid !=
 768             oi->inp_cred->cr_uid))
 769                 return (0);
 770
 771         /* Check the existing inp has BINDMULTI set */
 772         if ((ni->inp_flags2 & INP_BINDMULTI) &&
 773             ((oi->inp_flags2 & INP_BINDMULTI) == 0))
 774                 return (0);
 775
 776         /*
 777          * We're okay - either INP_BINDMULTI isn't set on ni, or
 778          * it is and it matches the checks.
 779          */
 780         return (1);
 781 }
 782
 783 #ifdef INET
 784 /*
 785  * Set up a bind operation on a PCB, performing port allocation
 786  * as required, but do not actually modify the PCB. Callers can
 787  * either complete the bind by setting inp_laddr/inp_lport and
 788  * calling in_pcbinshash(), or they can just use the resulting
 789  * port and address to authorise the sending of a once-off packet.
 790  *
 791  * On error, the values of *laddrp and *lportp are not changed.
 792  */
 793 int
 794 in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp,
 795     u_short *lportp, struct ucred *cred)
 796 {
 797         struct socket *so = inp->inp_socket;
 798         struct sockaddr_in *sin;
 799         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 800         struct in_addr laddr;
 801         u_short lport = 0;
 802         int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
 803         int error;
 804
 805         /*
 806          * XXX: Maybe we could let SO_REUSEPORT_LB set SO_REUSEPORT bit here
 807          * so that we don't have to add to the (already messy) code below.
 808          */
 809         int reuseport_lb = (so->so_options & SO_REUSEPORT_LB);
 810
 811         /*
 812          * No state changes, so read locks are sufficient here.
 813          */
 814         INP_LOCK_ASSERT(inp);
 815         INP_HASH_LOCK_ASSERT(pcbinfo);
 816
 817         if (CK_STAILQ_EMPTY(&V_in_ifaddrhead)) /* XXX broken! */
 818                 return (EADDRNOTAVAIL);
 819         laddr.s_addr = *laddrp;
 820         if (nam != NULL && laddr.s_addr != INADDR_ANY)
 821                 return (EINVAL);
 822         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
 823                 lookupflags = INPLOOKUP_WILDCARD;
 824         if (nam == NULL) {
 825                 if ((error = prison_local_ip4(cred, &laddr)) != 0)
 826                         return (error);
 827         } else {
 828                 sin = (struct sockaddr_in *)nam;
 829                 if (nam->sa_len != sizeof (*sin))
 830                         return (EINVAL);
 831 #ifdef notdef
 832                 /*
 833                  * We should check the family, but old programs
 834                  * incorrectly fail to initialize it.
 835                  */
 836                 if (sin->sin_family != AF_INET)
 837                         return (EAFNOSUPPORT);
 838 #endif
 839                 error = prison_local_ip4(cred, &sin->sin_addr);
 840                 if (error)
 841                         return (error);
 842                 if (sin->sin_port != *lportp) {
 843                         /* Don't allow the port to change. */
 844                         if (*lportp != 0)
 845                                 return (EINVAL);
 846                         lport = sin->sin_port;
 847                 }
 848                 /* NB: lport is left as 0 if the port isn't being changed. */
 849                 if (IN_MULTICAST(ntohl(sin->sin_addr.s_addr))) {
 850                         /*
 851                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 852                          * allow complete duplication of binding if
 853                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 854                          * and a multicast address is bound on both
 855                          * new and duplicated sockets.
 856                          */
 857                         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
 858                                 reuseport = SO_REUSEADDR|SO_REUSEPORT;
 859                         /*
 860                          * XXX: How to deal with SO_REUSEPORT_LB here?
 861                          * Treat same as SO_REUSEPORT for now.
 862                          */
 863                         if ((so->so_options &
 864                             (SO_REUSEADDR|SO_REUSEPORT_LB)) != 0)
 865                                 reuseport_lb = SO_REUSEADDR|SO_REUSEPORT_LB;
 866                 } else if (sin->sin_addr.s_addr != INADDR_ANY) {
 867                         sin->sin_port = 0;              /* yech... */
 868                         bzero(&sin->sin_zero, sizeof(sin->sin_zero));
 869                         /*
 870                          * Is the address a local IP address?
 871                          * If INP_BINDANY is set, then the socket may be bound
 872                          * to any endpoint address, local or not.
 873                          */
 874                         if ((inp->inp_flags & INP_BINDANY) == 0 &&
 875                             ifa_ifwithaddr_check((struct sockaddr *)sin) == 0)
 876                                 return (EADDRNOTAVAIL);
 877                 }
 878                 laddr = sin->sin_addr;
 879                 if (lport) {
 880                         struct inpcb *t;
 881                         struct tcptw *tw;
 882
 883                         /* GROSS */
 884                         if (ntohs(lport) <= V_ipport_reservedhigh &&
 885                             ntohs(lport) >= V_ipport_reservedlow &&
 886                             priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
 887                                 return (EACCES);
 888                         if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
 889                             priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
 890                                 t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 891                                     lport, INPLOOKUP_WILDCARD, cred);
 892         /*
 893          * XXX
 894          * This entire block sorely needs a rewrite.
 895          */
 896                                 if (t &&
 897                                     ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 898                                     ((t->inp_flags & INP_TIMEWAIT) == 0) &&
 899                                     (so->so_type != SOCK_STREAM ||
 900                                      ntohl(t->inp_faddr.s_addr) == INADDR_ANY) &&
 901                                     (ntohl(sin->sin_addr.s_addr) != INADDR_ANY ||
 902                                      ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
 903                                      (t->inp_flags2 & INP_REUSEPORT) ||
 904                                      (t->inp_flags2 & INP_REUSEPORT_LB) == 0) &&
 905                                     (inp->inp_cred->cr_uid !=
 906                                      t->inp_cred->cr_uid))
 907                                         return (EADDRINUSE);
 908
 909                                 /*
 910                                  * If the socket is a BINDMULTI socket, then
 911                                  * the credentials need to match and the
 912                                  * original socket also has to have been bound
 913                                  * with BINDMULTI.
 914                                  */
 915                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 916                                         return (EADDRINUSE);
 917                         }
 918                         t = in_pcblookup_local(pcbinfo, sin->sin_addr,
 919                             lport, lookupflags, cred);
 920                         if (t && (t->inp_flags & INP_TIMEWAIT)) {
 921                                 /*
 922                                  * XXXRW: If an incpb has had its timewait
 923                                  * state recycled, we treat the address as
 924                                  * being in use (for now).  This is better
 925                                  * than a panic, but not desirable.
 926                                  */
 927                                 tw = intotw(t);
 928                                 if (tw == NULL ||
 929                                     ((reuseport & tw->tw_so_options) == 0 &&
 930                                         (reuseport_lb &
 931                                             tw->tw_so_options) == 0)) {
 932                                         return (EADDRINUSE);
 933                                 }
 934                         } else if (t &&
 935                                    ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 936                                    (reuseport & inp_so_options(t)) == 0 &&
 937                                    (reuseport_lb & inp_so_options(t)) == 0) {
 938 #ifdef INET6
 939                                 if (ntohl(sin->sin_addr.s_addr) !=
 940                                     INADDR_ANY ||
 941                                     ntohl(t->inp_laddr.s_addr) !=
 942                                     INADDR_ANY ||
 943                                     (inp->inp_vflag & INP_IPV6PROTO) == 0 ||
 944                                     (t->inp_vflag & INP_IPV6PROTO) == 0)
 945 #endif
 946                                                 return (EADDRINUSE);
 947                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 948                                         return (EADDRINUSE);
 949                         }
 950                 }
 951         }
 952         if (*lportp != 0)
 953                 lport = *lportp;
 954         if (lport == 0) {
 955                 error = in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
 956                 if (error != 0)
 957                         return (error);
 958
 959         }
 960         *laddrp = laddr.s_addr;
 961         *lportp = lport;
 962         return (0);
 963 }
 964
 965 /*
 966  * Connect from a socket to a specified address.
 967  * Both address and port must be specified in argument sin.
 968  * If don't have a local address for this socket yet,
 969  * then pick one.
 970  */
 971 int
 972 in_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
 973     struct ucred *cred, struct mbuf *m, bool rehash)
 974 {
 975         u_short lport, fport;
 976         in_addr_t laddr, faddr;
 977         int anonport, error;
 978
 979         INP_WLOCK_ASSERT(inp);
 980         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 981
 982         lport = inp->inp_lport;
 983         laddr = inp->inp_laddr.s_addr;
 984         anonport = (lport == 0);
 985         error = in_pcbconnect_setup(inp, nam, &laddr, &lport, &faddr, &fport,
 986             NULL, cred);
 987         if (error)
 988                 return (error);
 989
 990         /* Do the initial binding of the local address if required. */
 991         if (inp->inp_laddr.s_addr == INADDR_ANY && inp->inp_lport == 0) {
 992                 KASSERT(rehash == true,
 993                     ("Rehashing required for unbound inps"));
 994                 inp->inp_lport = lport;
 995                 inp->inp_laddr.s_addr = laddr;
 996                 if (in_pcbinshash(inp) != 0) {
 997                         inp->inp_laddr.s_addr = INADDR_ANY;
 998                         inp->inp_lport = 0;
 999                         return (EAGAIN);
1000                 }
1001         }
1002
1003         /* Commit the remaining changes. */
1004         inp->inp_lport = lport;
1005         inp->inp_laddr.s_addr = laddr;
1006         inp->inp_faddr.s_addr = faddr;
1007         inp->inp_fport = fport;
1008         if (rehash) {
1009                 in_pcbrehash_mbuf(inp, m);
1010         } else {
1011                 in_pcbinshash_mbuf(inp, m);
1012         }
1013
1014         if (anonport)
1015                 inp->inp_flags |= INP_ANONPORT;
1016         return (0);
1017 }
1018
1019 int
1020 in_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
1021 {
1022
1023         return (in_pcbconnect_mbuf(inp, nam, cred, NULL, true));
1024 }
1025
1026 /*
1027  * Do proper source address selection on an unbound socket in case
1028  * of connect. Take jails into account as well.
1029  */
1030 int
1031 in_pcbladdr(struct inpcb *inp, struct in_addr *faddr, struct in_addr *laddr,
1032     struct ucred *cred)
1033 {
1034         struct ifaddr *ifa;
1035         struct sockaddr *sa;
1036         struct sockaddr_in *sin;
1037         struct route sro;
1038         int error;
1039
1040         NET_EPOCH_ASSERT();
1041         KASSERT(laddr != NULL, ("%s: laddr NULL", __func__));
1042         /*
1043          * Bypass source address selection and use the primary jail IP
1044          * if requested.
1045          */
1046         if (cred != NULL && !prison_saddrsel_ip4(cred, laddr))
1047                 return (0);
1048
1049         error = 0;
1050         bzero(&sro, sizeof(sro));
1051
1052         sin = (struct sockaddr_in *)&sro.ro_dst;
1053         sin->sin_family = AF_INET;
1054         sin->sin_len = sizeof(struct sockaddr_in);
1055         sin->sin_addr.s_addr = faddr->s_addr;
1056
1057         /*
1058          * If route is known our src addr is taken from the i/f,
1059          * else punt.
1060          *
1061          * Find out route to destination.
1062          */
1063         if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0)
1064                 in_rtalloc_ign(&sro, 0, inp->inp_inc.inc_fibnum);
1065
1066         /*
1067          * If we found a route, use the address corresponding to
1068          * the outgoing interface.
1069          *
1070          * Otherwise assume faddr is reachable on a directly connected
1071          * network and try to find a corresponding interface to take
1072          * the source address from.
1073          */
1074         if (sro.ro_rt == NULL || sro.ro_rt->rt_ifp == NULL) {
1075                 struct in_ifaddr *ia;
1076                 struct ifnet *ifp;
1077
1078                 ia = ifatoia(ifa_ifwithdstaddr((struct sockaddr *)sin,
1079                                         inp->inp_socket->so_fibnum));
1080                 if (ia == NULL) {
1081                         ia = ifatoia(ifa_ifwithnet((struct sockaddr *)sin, 0,
1082                                                 inp->inp_socket->so_fibnum));
1083
1084                 }
1085                 if (ia == NULL) {
1086                         error = ENETUNREACH;
1087                         goto done;
1088                 }
1089
1090                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1091                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1092                         goto done;
1093                 }
1094
1095                 ifp = ia->ia_ifp;
1096                 ia = NULL;
1097                 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1098
1099                         sa = ifa->ifa_addr;
1100                         if (sa->sa_family != AF_INET)
1101                                 continue;
1102                         sin = (struct sockaddr_in *)sa;
1103                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1104                                 ia = (struct in_ifaddr *)ifa;
1105                                 break;
1106                         }
1107                 }
1108                 if (ia != NULL) {
1109                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1110                         goto done;
1111                 }
1112
1113                 /* 3. As a last resort return the 'default' jail address. */
1114                 error = prison_get_ip4(cred, laddr);
1115                 goto done;
1116         }
1117
1118         /*
1119          * If the outgoing interface on the route found is not
1120          * a loopback interface, use the address from that interface.
1121          * In case of jails do those three steps:
1122          * 1. check if the interface address belongs to the jail. If so use it.
1123          * 2. check if we have any address on the outgoing interface
1124          *    belonging to this jail. If so use it.
1125          * 3. as a last resort return the 'default' jail address.
1126          */
1127         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) == 0) {
1128                 struct in_ifaddr *ia;
1129                 struct ifnet *ifp;
1130
1131                 /* If not jailed, use the default returned. */
1132                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1133                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
1134                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1135                         goto done;
1136                 }
1137
1138                 /* Jailed. */
1139                 /* 1. Check if the iface address belongs to the jail. */
1140                 sin = (struct sockaddr_in *)sro.ro_rt->rt_ifa->ifa_addr;
1141                 if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1142                         ia = (struct in_ifaddr *)sro.ro_rt->rt_ifa;
1143                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1144                         goto done;
1145                 }
1146
1147                 /*
1148                  * 2. Check if we have any address on the outgoing interface
1149                  *    belonging to this jail.
1150                  */
1151                 ia = NULL;
1152                 ifp = sro.ro_rt->rt_ifp;
1153                 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1154                         sa = ifa->ifa_addr;
1155                         if (sa->sa_family != AF_INET)
1156                                 continue;
1157                         sin = (struct sockaddr_in *)sa;
1158                         if (prison_check_ip4(cred, &sin->sin_addr) == 0) {
1159                                 ia = (struct in_ifaddr *)ifa;
1160                                 break;
1161                         }
1162                 }
1163                 if (ia != NULL) {
1164                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1165                         goto done;
1166                 }
1167
1168                 /* 3. As a last resort return the 'default' jail address. */
1169                 error = prison_get_ip4(cred, laddr);
1170                 goto done;
1171         }
1172
1173         /*
1174          * The outgoing interface is marked with 'loopback net', so a route
1175          * to ourselves is here.
1176          * Try to find the interface of the destination address and then
1177          * take the address from there. That interface is not necessarily
1178          * a loopback interface.
1179          * In case of jails, check that it is an address of the jail
1180          * and if we cannot find, fall back to the 'default' jail address.
1181          */
1182         if ((sro.ro_rt->rt_ifp->if_flags & IFF_LOOPBACK) != 0) {
1183                 struct sockaddr_in sain;
1184                 struct in_ifaddr *ia;
1185
1186                 bzero(&sain, sizeof(struct sockaddr_in));
1187                 sain.sin_family = AF_INET;
1188                 sain.sin_len = sizeof(struct sockaddr_in);
1189                 sain.sin_addr.s_addr = faddr->s_addr;
1190
1191                 ia = ifatoia(ifa_ifwithdstaddr(sintosa(&sain),
1192                                         inp->inp_socket->so_fibnum));
1193                 if (ia == NULL)
1194                         ia = ifatoia(ifa_ifwithnet(sintosa(&sain), 0,
1195                                                 inp->inp_socket->so_fibnum));
1196                 if (ia == NULL)
1197                         ia = ifatoia(ifa_ifwithaddr(sintosa(&sain)));
1198
1199                 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1200                         if (ia == NULL) {
1201                                 error = ENETUNREACH;
1202                                 goto done;
1203                         }
1204                         laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1205                         goto done;
1206                 }
1207
1208                 /* Jailed. */
1209                 if (ia != NULL) {
1210                         struct ifnet *ifp;
1211
1212                         ifp = ia->ia_ifp;
1213                         ia = NULL;
1214                         CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1215                                 sa = ifa->ifa_addr;
1216                                 if (sa->sa_family != AF_INET)
1217                                         continue;
1218                                 sin = (struct sockaddr_in *)sa;
1219                                 if (prison_check_ip4(cred,
1220                                     &sin->sin_addr) == 0) {
1221                                         ia = (struct in_ifaddr *)ifa;
1222                                         break;
1223                                 }
1224                         }
1225                         if (ia != NULL) {
1226                                 laddr->s_addr = ia->ia_addr.sin_addr.s_addr;
1227                                 goto done;
1228                         }
1229                 }
1230
1231                 /* 3. As a last resort return the 'default' jail address. */
1232                 error = prison_get_ip4(cred, laddr);
1233                 goto done;
1234         }
1235
1236 done:
1237         if (sro.ro_rt != NULL)
1238                 RTFREE(sro.ro_rt);
1239         return (error);
1240 }
1241
1242 /*
1243  * Set up for a connect from a socket to the specified address.
1244  * On entry, *laddrp and *lportp should contain the current local
1245  * address and port for the PCB; these are updated to the values
1246  * that should be placed in inp_laddr and inp_lport to complete
1247  * the connect.
1248  *
1249  * On success, *faddrp and *fportp will be set to the remote address
1250  * and port. These are not updated in the error case.
1251  *
1252  * If the operation fails because the connection already exists,
1253  * *oinpp will be set to the PCB of that connection so that the
1254  * caller can decide to override it. In all other cases, *oinpp
1255  * is set to NULL.
1256  */
1257 int
1258 in_pcbconnect_setup(struct inpcb *inp, struct sockaddr *nam,
1259     in_addr_t *laddrp, u_short *lportp, in_addr_t *faddrp, u_short *fportp,
1260     struct inpcb **oinpp, struct ucred *cred)
1261 {
1262         struct rm_priotracker in_ifa_tracker;
1263         struct sockaddr_in *sin = (struct sockaddr_in *)nam;
1264         struct in_ifaddr *ia;
1265         struct inpcb *oinp;
1266         struct in_addr laddr, faddr;
1267         u_short lport, fport;
1268         int error;
1269
1270         /*
1271          * Because a global state change doesn't actually occur here, a read
1272          * lock is sufficient.
1273          */
1274         NET_EPOCH_ASSERT();
1275         INP_LOCK_ASSERT(inp);
1276         INP_HASH_LOCK_ASSERT(inp->inp_pcbinfo);
1277
1278         if (oinpp != NULL)
1279                 *oinpp = NULL;
1280         if (nam->sa_len != sizeof (*sin))
1281                 return (EINVAL);
1282         if (sin->sin_family != AF_INET)
1283                 return (EAFNOSUPPORT);
1284         if (sin->sin_port == 0)
1285                 return (EADDRNOTAVAIL);
1286         laddr.s_addr = *laddrp;
1287         lport = *lportp;
1288         faddr = sin->sin_addr;
1289         fport = sin->sin_port;
1290
1291         if (!CK_STAILQ_EMPTY(&V_in_ifaddrhead)) {
1292                 /*
1293                  * If the destination address is INADDR_ANY,
1294                  * use the primary local address.
1295                  * If the supplied address is INADDR_BROADCAST,
1296                  * and the primary interface supports broadcast,
1297                  * choose the broadcast address for that interface.
1298                  */
1299                 if (faddr.s_addr == INADDR_ANY) {
1300                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1301                         faddr =
1302                             IA_SIN(CK_STAILQ_FIRST(&V_in_ifaddrhead))->sin_addr;
1303                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1304                         if (cred != NULL &&
1305                             (error = prison_get_ip4(cred, &faddr)) != 0)
1306                                 return (error);
1307                 } else if (faddr.s_addr == (u_long)INADDR_BROADCAST) {
1308                         IN_IFADDR_RLOCK(&in_ifa_tracker);
1309                         if (CK_STAILQ_FIRST(&V_in_ifaddrhead)->ia_ifp->if_flags &
1310                             IFF_BROADCAST)
1311                                 faddr = satosin(&CK_STAILQ_FIRST(
1312                                     &V_in_ifaddrhead)->ia_broadaddr)->sin_addr;
1313                         IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1314                 }
1315         }
1316         if (laddr.s_addr == INADDR_ANY) {
1317                 error = in_pcbladdr(inp, &faddr, &laddr, cred);
1318                 /*
1319                  * If the destination address is multicast and an outgoing
1320                  * interface has been set as a multicast option, prefer the
1321                  * address of that interface as our source address.
1322                  */
1323                 if (IN_MULTICAST(ntohl(faddr.s_addr)) &&
1324                     inp->inp_moptions != NULL) {
1325                         struct ip_moptions *imo;
1326                         struct ifnet *ifp;
1327
1328                         imo = inp->inp_moptions;
1329                         if (imo->imo_multicast_ifp != NULL) {
1330                                 ifp = imo->imo_multicast_ifp;
1331                                 IN_IFADDR_RLOCK(&in_ifa_tracker);
1332                                 CK_STAILQ_FOREACH(ia, &V_in_ifaddrhead, ia_link) {
1333                                         if ((ia->ia_ifp == ifp) &&
1334                                             (cred == NULL ||
1335                                             prison_check_ip4(cred,
1336                                             &ia->ia_addr.sin_addr) == 0))
1337                                                 break;
1338                                 }
1339                                 if (ia == NULL)
1340                                         error = EADDRNOTAVAIL;
1341                                 else {
1342                                         laddr = ia->ia_addr.sin_addr;
1343                                         error = 0;
1344                                 }
1345                                 IN_IFADDR_RUNLOCK(&in_ifa_tracker);
1346                         }
1347                 }
1348                 if (error)
1349                         return (error);
1350         }
1351         oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr, fport,
1352             laddr, lport, 0, NULL);
1353         if (oinp != NULL) {
1354                 if (oinpp != NULL)
1355                         *oinpp = oinp;
1356                 return (EADDRINUSE);
1357         }
1358         if (lport == 0) {
1359                 error = in_pcbbind_setup(inp, NULL, &laddr.s_addr, &lport,
1360                     cred);
1361                 if (error)
1362                         return (error);
1363         }
1364         *laddrp = laddr.s_addr;
1365         *lportp = lport;
1366         *faddrp = faddr.s_addr;
1367         *fportp = fport;
1368         return (0);
1369 }
1370
1371 void
1372 in_pcbdisconnect(struct inpcb *inp)
1373 {
1374
1375         INP_WLOCK_ASSERT(inp);
1376         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
1377
1378         inp->inp_faddr.s_addr = INADDR_ANY;
1379         inp->inp_fport = 0;
1380         in_pcbrehash(inp);
1381 }
1382 #endif /* INET */
1383
1384 /*
1385  * in_pcbdetach() is responsibe for disassociating a socket from an inpcb.
1386  * For most protocols, this will be invoked immediately prior to calling
1387  * in_pcbfree().  However, with TCP the inpcb may significantly outlive the
1388  * socket, in which case in_pcbfree() is deferred.
1389  */
1390 void
1391 in_pcbdetach(struct inpcb *inp)
1392 {
1393
1394         KASSERT(inp->inp_socket != NULL, ("%s: inp_socket == NULL", __func__));
1395
1396 #ifdef RATELIMIT
1397         if (inp->inp_snd_tag != NULL)
1398                 in_pcbdetach_txrtlmt(inp);
1399 #endif
1400         inp->inp_socket->so_pcb = NULL;
1401         inp->inp_socket = NULL;
1402 }
1403
1404 /*
1405  * in_pcbref() bumps the reference count on an inpcb in order to maintain
1406  * stability of an inpcb pointer despite the inpcb lock being released.  This
1407  * is used in TCP when the inpcbinfo lock needs to be acquired or upgraded,
1408  * but where the inpcb lock may already held, or when acquiring a reference
1409  * via a pcbgroup.
1410  *
1411  * in_pcbref() should be used only to provide brief memory stability, and
1412  * must always be followed by a call to INP_WLOCK() and in_pcbrele() to
1413  * garbage collect the inpcb if it has been in_pcbfree()'d from another
1414  * context.  Until in_pcbrele() has returned that the inpcb is still valid,
1415  * lock and rele are the *only* safe operations that may be performed on the
1416  * inpcb.
1417  *
1418  * While the inpcb will not be freed, releasing the inpcb lock means that the
1419  * connection's state may change, so the caller should be careful to
1420  * revalidate any cached state on reacquiring the lock.  Drop the reference
1421  * using in_pcbrele().
1422  */
1423 void
1424 in_pcbref(struct inpcb *inp)
1425 {
1426
1427         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1428
1429         refcount_acquire(&inp->inp_refcount);
1430 }
1431
1432 /*
1433  * Drop a refcount on an inpcb elevated using in_pcbref(); because a call to
1434  * in_pcbfree() may have been made between in_pcbref() and in_pcbrele(), we
1435  * return a flag indicating whether or not the inpcb remains valid.  If it is
1436  * valid, we return with the inpcb lock held.
1437  *
1438  * Notice that, unlike in_pcbref(), the inpcb lock must be held to drop a
1439  * reference on an inpcb.  Historically more work was done here (actually, in
1440  * in_pcbfree_internal()) but has been moved to in_pcbfree() to avoid the
1441  * need for the pcbinfo lock in in_pcbrele().  Deferring the free is entirely
1442  * about memory stability (and continued use of the write lock).
1443  */
1444 int
1445 in_pcbrele_rlocked(struct inpcb *inp)
1446 {
1447         struct inpcbinfo *pcbinfo;
1448
1449         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1450
1451         INP_RLOCK_ASSERT(inp);
1452
1453         if (refcount_release(&inp->inp_refcount) == 0) {
1454                 /*
1455                  * If the inpcb has been freed, let the caller know, even if
1456                  * this isn't the last reference.
1457                  */
1458                 if (inp->inp_flags2 & INP_FREED) {
1459                         INP_RUNLOCK(inp);
1460                         return (1);
1461                 }
1462                 return (0);
1463         }
1464
1465         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1466 #ifdef TCPHPTS
1467         if (inp->inp_in_hpts || inp->inp_in_input) {
1468                 struct tcp_hpts_entry *hpts;
1469                 /*
1470                  * We should not be on the hpts at
1471                  * this point in any form. we must
1472                  * get the lock to be sure.
1473                  */
1474                 hpts = tcp_hpts_lock(inp);
1475                 if (inp->inp_in_hpts)
1476                         panic("Hpts:%p inp:%p at free still on hpts",
1477                               hpts, inp);
1478                 mtx_unlock(&hpts->p_mtx);
1479                 hpts = tcp_input_lock(inp);
1480                 if (inp->inp_in_input)
1481                         panic("Hpts:%p inp:%p at free still on input hpts",
1482                               hpts, inp);
1483                 mtx_unlock(&hpts->p_mtx);
1484         }
1485 #endif
1486         INP_RUNLOCK(inp);
1487         pcbinfo = inp->inp_pcbinfo;
1488         uma_zfree(pcbinfo->ipi_zone, inp);
1489         return (1);
1490 }
1491
1492 int
1493 in_pcbrele_wlocked(struct inpcb *inp)
1494 {
1495         struct inpcbinfo *pcbinfo;
1496
1497         KASSERT(inp->inp_refcount > 0, ("%s: refcount 0", __func__));
1498
1499         INP_WLOCK_ASSERT(inp);
1500
1501         if (refcount_release(&inp->inp_refcount) == 0) {
1502                 /*
1503                  * If the inpcb has been freed, let the caller know, even if
1504                  * this isn't the last reference.
1505                  */
1506                 if (inp->inp_flags2 & INP_FREED) {
1507                         INP_WUNLOCK(inp);
1508                         return (1);
1509                 }
1510                 return (0);
1511         }
1512
1513         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1514 #ifdef TCPHPTS
1515         if (inp->inp_in_hpts || inp->inp_in_input) {
1516                 struct tcp_hpts_entry *hpts;
1517                 /*
1518                  * We should not be on the hpts at
1519                  * this point in any form. we must
1520                  * get the lock to be sure.
1521                  */
1522                 hpts = tcp_hpts_lock(inp);
1523                 if (inp->inp_in_hpts)
1524                         panic("Hpts:%p inp:%p at free still on hpts",
1525                               hpts, inp);
1526                 mtx_unlock(&hpts->p_mtx);
1527                 hpts = tcp_input_lock(inp);
1528                 if (inp->inp_in_input)
1529                         panic("Hpts:%p inp:%p at free still on input hpts",
1530                               hpts, inp);
1531                 mtx_unlock(&hpts->p_mtx);
1532         }
1533 #endif
1534         INP_WUNLOCK(inp);
1535         pcbinfo = inp->inp_pcbinfo;
1536         uma_zfree(pcbinfo->ipi_zone, inp);
1537         return (1);
1538 }
1539
1540 /*
1541  * Temporary wrapper.
1542  */
1543 int
1544 in_pcbrele(struct inpcb *inp)
1545 {
1546
1547         return (in_pcbrele_wlocked(inp));
1548 }
1549
1550 void
1551 in_pcblist_rele_rlocked(epoch_context_t ctx)
1552 {
1553         struct in_pcblist *il;
1554         struct inpcb *inp;
1555         struct inpcbinfo *pcbinfo;
1556         int i, n;
1557
1558         il = __containerof(ctx, struct in_pcblist, il_epoch_ctx);
1559         pcbinfo = il->il_pcbinfo;
1560         n = il->il_count;
1561         INP_INFO_WLOCK(pcbinfo);
1562         for (i = 0; i < n; i++) {
1563                 inp = il->il_inp_list[i];
1564                 INP_RLOCK(inp);
1565                 if (!in_pcbrele_rlocked(inp))
1566                         INP_RUNLOCK(inp);
1567         }
1568         INP_INFO_WUNLOCK(pcbinfo);
1569         free(il, M_TEMP);
1570 }
1571
1572 static void
1573 inpcbport_free(epoch_context_t ctx)
1574 {
1575         struct inpcbport *phd;
1576
1577         phd = __containerof(ctx, struct inpcbport, phd_epoch_ctx);
1578         free(phd, M_PCB);
1579 }
1580
1581 static void
1582 in_pcbfree_deferred(epoch_context_t ctx)
1583 {
1584         struct inpcb *inp;
1585         int released __unused;
1586
1587         inp = __containerof(ctx, struct inpcb, inp_epoch_ctx);
1588
1589         INP_WLOCK(inp);
1590         CURVNET_SET(inp->inp_vnet);
1591 #ifdef INET
1592         struct ip_moptions *imo = inp->inp_moptions;
1593         inp->inp_moptions = NULL;
1594 #endif
1595         /* XXXRW: Do as much as possible here. */
1596 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
1597         if (inp->inp_sp != NULL)
1598                 ipsec_delete_pcbpolicy(inp);
1599 #endif
1600 #ifdef INET6
1601         struct ip6_moptions *im6o = NULL;
1602         if (inp->inp_vflag & INP_IPV6PROTO) {
1603                 ip6_freepcbopts(inp->in6p_outputopts);
1604                 im6o = inp->in6p_moptions;
1605                 inp->in6p_moptions = NULL;
1606         }
1607 #endif
1608         if (inp->inp_options)
1609                 (void)m_free(inp->inp_options);
1610         inp->inp_vflag = 0;
1611         crfree(inp->inp_cred);
1612 #ifdef MAC
1613         mac_inpcb_destroy(inp);
1614 #endif
1615         released = in_pcbrele_wlocked(inp);
1616         MPASS(released);
1617 #ifdef INET6
1618         ip6_freemoptions(im6o);
1619 #endif
1620 #ifdef INET
1621         inp_freemoptions(imo);
1622 #endif
1623         CURVNET_RESTORE();
1624 }
1625
1626 /*
1627  * Unconditionally schedule an inpcb to be freed by decrementing its
1628  * reference count, which should occur only after the inpcb has been detached
1629  * from its socket.  If another thread holds a temporary reference (acquired
1630  * using in_pcbref()) then the free is deferred until that reference is
1631  * released using in_pcbrele(), but the inpcb is still unlocked.  Almost all
1632  * work, including removal from global lists, is done in this context, where
1633  * the pcbinfo lock is held.
1634  */
1635 void
1636 in_pcbfree(struct inpcb *inp)
1637 {
1638         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
1639
1640         KASSERT(inp->inp_socket == NULL, ("%s: inp_socket != NULL", __func__));
1641         KASSERT((inp->inp_flags2 & INP_FREED) == 0,
1642             ("%s: called twice for pcb %p", __func__, inp));
1643         if (inp->inp_flags2 & INP_FREED) {
1644                 INP_WUNLOCK(inp);
1645                 return;
1646         }
1647
1648         INP_WLOCK_ASSERT(inp);
1649         INP_LIST_WLOCK(pcbinfo);
1650         in_pcbremlists(inp);
1651         INP_LIST_WUNLOCK(pcbinfo);
1652         RO_INVALIDATE_CACHE(&inp->inp_route);
1653         /* mark as destruction in progress */
1654         inp->inp_flags2 |= INP_FREED;
1655         INP_WUNLOCK(inp);
1656         NET_EPOCH_CALL(in_pcbfree_deferred, &inp->inp_epoch_ctx);
1657 }
1658
1659 /*
1660  * in_pcbdrop() removes an inpcb from hashed lists, releasing its address and
1661  * port reservation, and preventing it from being returned by inpcb lookups.
1662  *
1663  * It is used by TCP to mark an inpcb as unused and avoid future packet
1664  * delivery or event notification when a socket remains open but TCP has
1665  * closed.  This might occur as a result of a shutdown()-initiated TCP close
1666  * or a RST on the wire, and allows the port binding to be reused while still
1667  * maintaining the invariant that so_pcb always points to a valid inpcb until
1668  * in_pcbdetach().
1669  *
1670  * XXXRW: Possibly in_pcbdrop() should also prevent future notifications by
1671  * in_pcbnotifyall() and in_pcbpurgeif0()?
1672  */
1673 void
1674 in_pcbdrop(struct inpcb *inp)
1675 {
1676
1677         INP_WLOCK_ASSERT(inp);
1678 #ifdef INVARIANTS
1679         if (inp->inp_socket != NULL && inp->inp_ppcb != NULL)
1680                 MPASS(inp->inp_refcount > 1);
1681 #endif
1682
1683         /*
1684          * XXXRW: Possibly we should protect the setting of INP_DROPPED with
1685          * the hash lock...?
1686          */
1687         inp->inp_flags |= INP_DROPPED;
1688         if (inp->inp_flags & INP_INHASHLIST) {
1689                 struct inpcbport *phd = inp->inp_phd;
1690
1691                 INP_HASH_WLOCK(inp->inp_pcbinfo);
1692                 in_pcbremlbgrouphash(inp);
1693                 CK_LIST_REMOVE(inp, inp_hash);
1694                 CK_LIST_REMOVE(inp, inp_portlist);
1695                 if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
1696                         CK_LIST_REMOVE(phd, phd_hash);
1697                         NET_EPOCH_CALL(inpcbport_free, &phd->phd_epoch_ctx);
1698                 }
1699                 INP_HASH_WUNLOCK(inp->inp_pcbinfo);
1700                 inp->inp_flags &= ~INP_INHASHLIST;
1701 #ifdef PCBGROUP
1702                 in_pcbgroup_remove(inp);
1703 #endif
1704         }
1705 }
1706
1707 #ifdef INET
1708 /*
1709  * Common routines to return the socket addresses associated with inpcbs.
1710  */
1711 struct sockaddr *
1712 in_sockaddr(in_port_t port, struct in_addr *addr_p)
1713 {
1714         struct sockaddr_in *sin;
1715
1716         sin = malloc(sizeof *sin, M_SONAME,
1717                 M_WAITOK | M_ZERO);
1718         sin->sin_family = AF_INET;
1719         sin->sin_len = sizeof(*sin);
1720         sin->sin_addr = *addr_p;
1721         sin->sin_port = port;
1722
1723         return (struct sockaddr *)sin;
1724 }
1725
1726 int
1727 in_getsockaddr(struct socket *so, struct sockaddr **nam)
1728 {
1729         struct inpcb *inp;
1730         struct in_addr addr;
1731         in_port_t port;
1732
1733         inp = sotoinpcb(so);
1734         KASSERT(inp != NULL, ("in_getsockaddr: inp == NULL"));
1735
1736         INP_RLOCK(inp);
1737         port = inp->inp_lport;
1738         addr = inp->inp_laddr;
1739         INP_RUNLOCK(inp);
1740
1741         *nam = in_sockaddr(port, &addr);
1742         return 0;
1743 }
1744
1745 int
1746 in_getpeeraddr(struct socket *so, struct sockaddr **nam)
1747 {
1748         struct inpcb *inp;
1749         struct in_addr addr;
1750         in_port_t port;
1751
1752         inp = sotoinpcb(so);
1753         KASSERT(inp != NULL, ("in_getpeeraddr: inp == NULL"));
1754
1755         INP_RLOCK(inp);
1756         port = inp->inp_fport;
1757         addr = inp->inp_faddr;
1758         INP_RUNLOCK(inp);
1759
1760         *nam = in_sockaddr(port, &addr);
1761         return 0;
1762 }
1763
1764 void
1765 in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr faddr, int errno,
1766     struct inpcb *(*notify)(struct inpcb *, int))
1767 {
1768         struct inpcb *inp, *inp_temp;
1769
1770         INP_INFO_WLOCK(pcbinfo);
1771         CK_LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
1772                 INP_WLOCK(inp);
1773 #ifdef INET6
1774                 if ((inp->inp_vflag & INP_IPV4) == 0) {
1775                         INP_WUNLOCK(inp);
1776                         continue;
1777                 }
1778 #endif
1779                 if (inp->inp_faddr.s_addr != faddr.s_addr ||
1780                     inp->inp_socket == NULL) {
1781                         INP_WUNLOCK(inp);
1782                         continue;
1783                 }
1784                 if ((*notify)(inp, errno))
1785                         INP_WUNLOCK(inp);
1786         }
1787         INP_INFO_WUNLOCK(pcbinfo);
1788 }
1789
1790 void
1791 in_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
1792 {
1793         struct inpcb *inp;
1794         struct in_multi *inm;
1795         struct in_mfilter *imf;
1796         struct ip_moptions *imo;
1797
1798         INP_INFO_WLOCK(pcbinfo);
1799         CK_LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
1800                 INP_WLOCK(inp);
1801                 imo = inp->inp_moptions;
1802                 if ((inp->inp_vflag & INP_IPV4) &&
1803                     imo != NULL) {
1804                         /*
1805                          * Unselect the outgoing interface if it is being
1806                          * detached.
1807                          */
1808                         if (imo->imo_multicast_ifp == ifp)
1809                                 imo->imo_multicast_ifp = NULL;
1810
1811                         /*
1812                          * Drop multicast group membership if we joined
1813                          * through the interface being detached.
1814                          *
1815                          * XXX This can all be deferred to an epoch_call
1816                          */
1817 restart:
1818                         IP_MFILTER_FOREACH(imf, &imo->imo_head) {
1819                                 if ((inm = imf->imf_inm) == NULL)
1820                                         continue;
1821                                 if (inm->inm_ifp != ifp)
1822                                         continue;
1823                                 ip_mfilter_remove(&imo->imo_head, imf);
1824                                 IN_MULTI_LOCK_ASSERT();
1825                                 in_leavegroup_locked(inm, NULL);
1826                                 ip_mfilter_free(imf);
1827                                 goto restart;
1828                         }
1829                 }
1830                 INP_WUNLOCK(inp);
1831         }
1832         INP_INFO_WUNLOCK(pcbinfo);
1833 }
1834
1835 /*
1836  * Lookup a PCB based on the local address and port.  Caller must hold the
1837  * hash lock.  No inpcb locks or references are acquired.
1838  */
1839 #define INP_LOOKUP_MAPPED_PCB_COST      3
1840 struct inpcb *
1841 in_pcblookup_local(struct inpcbinfo *pcbinfo, struct in_addr laddr,
1842     u_short lport, int lookupflags, struct ucred *cred)
1843 {
1844         struct inpcb *inp;
1845 #ifdef INET6
1846         int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
1847 #else
1848         int matchwild = 3;
1849 #endif
1850         int wildcard;
1851
1852         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1853             ("%s: invalid lookup flags %d", __func__, lookupflags));
1854
1855         INP_HASH_LOCK_ASSERT(pcbinfo);
1856
1857         if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
1858                 struct inpcbhead *head;
1859                 /*
1860                  * Look for an unconnected (wildcard foreign addr) PCB that
1861                  * matches the local address and port we're looking for.
1862                  */
1863                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
1864                     0, pcbinfo->ipi_hashmask)];
1865                 CK_LIST_FOREACH(inp, head, inp_hash) {
1866 #ifdef INET6
1867                         /* XXX inp locking */
1868                         if ((inp->inp_vflag & INP_IPV4) == 0)
1869                                 continue;
1870 #endif
1871                         if (inp->inp_faddr.s_addr == INADDR_ANY &&
1872                             inp->inp_laddr.s_addr == laddr.s_addr &&
1873                             inp->inp_lport == lport) {
1874                                 /*
1875                                  * Found?
1876                                  */
1877                                 if (cred == NULL ||
1878                                     prison_equal_ip4(cred->cr_prison,
1879                                         inp->inp_cred->cr_prison))
1880                                         return (inp);
1881                         }
1882                 }
1883                 /*
1884                  * Not found.
1885                  */
1886                 return (NULL);
1887         } else {
1888                 struct inpcbporthead *porthash;
1889                 struct inpcbport *phd;
1890                 struct inpcb *match = NULL;
1891                 /*
1892                  * Best fit PCB lookup.
1893                  *
1894                  * First see if this local port is in use by looking on the
1895                  * port hash list.
1896                  */
1897                 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
1898                     pcbinfo->ipi_porthashmask)];
1899                 CK_LIST_FOREACH(phd, porthash, phd_hash) {
1900                         if (phd->phd_port == lport)
1901                                 break;
1902                 }
1903                 if (phd != NULL) {
1904                         /*
1905                          * Port is in use by one or more PCBs. Look for best
1906                          * fit.
1907                          */
1908                         CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
1909                                 wildcard = 0;
1910                                 if (cred != NULL &&
1911                                     !prison_equal_ip4(inp->inp_cred->cr_prison,
1912                                         cred->cr_prison))
1913                                         continue;
1914 #ifdef INET6
1915                                 /* XXX inp locking */
1916                                 if ((inp->inp_vflag & INP_IPV4) == 0)
1917                                         continue;
1918                                 /*
1919                                  * We never select the PCB that has
1920                                  * INP_IPV6 flag and is bound to :: if
1921                                  * we have another PCB which is bound
1922                                  * to 0.0.0.0.  If a PCB has the
1923                                  * INP_IPV6 flag, then we set its cost
1924                                  * higher than IPv4 only PCBs.
1925                                  *
1926                                  * Note that the case only happens
1927                                  * when a socket is bound to ::, under
1928                                  * the condition that the use of the
1929                                  * mapped address is allowed.
1930                                  */
1931                                 if ((inp->inp_vflag & INP_IPV6) != 0)
1932                                         wildcard += INP_LOOKUP_MAPPED_PCB_COST;
1933 #endif
1934                                 if (inp->inp_faddr.s_addr != INADDR_ANY)
1935                                         wildcard++;
1936                                 if (inp->inp_laddr.s_addr != INADDR_ANY) {
1937                                         if (laddr.s_addr == INADDR_ANY)
1938                                                 wildcard++;
1939                                         else if (inp->inp_laddr.s_addr != laddr.s_addr)
1940                                                 continue;
1941                                 } else {
1942                                         if (laddr.s_addr != INADDR_ANY)
1943                                                 wildcard++;
1944                                 }
1945                                 if (wildcard < matchwild) {
1946                                         match = inp;
1947                                         matchwild = wildcard;
1948                                         if (matchwild == 0)
1949                                                 break;
1950                                 }
1951                         }
1952                 }
1953                 return (match);
1954         }
1955 }
1956 #undef INP_LOOKUP_MAPPED_PCB_COST
1957
1958 static struct inpcb *
1959 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
1960     const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr,
1961     uint16_t fport, int lookupflags)
1962 {
1963         struct inpcb *local_wild;
1964         const struct inpcblbgrouphead *hdr;
1965         struct inpcblbgroup *grp;
1966         uint32_t idx;
1967
1968         INP_HASH_LOCK_ASSERT(pcbinfo);
1969
1970         hdr = &pcbinfo->ipi_lbgrouphashbase[
1971             INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
1972
1973         /*
1974          * Order of socket selection:
1975          * 1. non-wild.
1976          * 2. wild (if lookupflags contains INPLOOKUP_WILDCARD).
1977          *
1978          * NOTE:
1979          * - Load balanced group does not contain jailed sockets
1980          * - Load balanced group does not contain IPv4 mapped INET6 wild sockets
1981          */
1982         local_wild = NULL;
1983         CK_LIST_FOREACH(grp, hdr, il_list) {
1984 #ifdef INET6
1985                 if (!(grp->il_vflag & INP_IPV4))
1986                         continue;
1987 #endif
1988                 if (grp->il_lport != lport)
1989                         continue;
1990
1991                 idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) %
1992                     grp->il_inpcnt;
1993                 if (grp->il_laddr.s_addr == laddr->s_addr)
1994                         return (grp->il_inp[idx]);
1995                 if (grp->il_laddr.s_addr == INADDR_ANY &&
1996                     (lookupflags & INPLOOKUP_WILDCARD) != 0)
1997                         local_wild = grp->il_inp[idx];
1998         }
1999         return (local_wild);
2000 }
2001
2002 #ifdef PCBGROUP
2003 /*
2004  * Lookup PCB in hash list, using pcbgroup tables.
2005  */
2006 static struct inpcb *
2007 in_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
2008     struct in_addr faddr, u_int fport_arg, struct in_addr laddr,
2009     u_int lport_arg, int lookupflags, struct ifnet *ifp)
2010 {
2011         struct inpcbhead *head;
2012         struct inpcb *inp, *tmpinp;
2013         u_short fport = fport_arg, lport = lport_arg;
2014         bool locked;
2015
2016         /*
2017          * First look for an exact match.
2018          */
2019         tmpinp = NULL;
2020         INP_GROUP_LOCK(pcbgroup);
2021         head = &pcbgroup->ipg_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
2022             pcbgroup->ipg_hashmask)];
2023         CK_LIST_FOREACH(inp, head, inp_pcbgrouphash) {
2024 #ifdef INET6
2025                 /* XXX inp locking */
2026                 if ((inp->inp_vflag & INP_IPV4) == 0)
2027                         continue;
2028 #endif
2029                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
2030                     inp->inp_laddr.s_addr == laddr.s_addr &&
2031                     inp->inp_fport == fport &&
2032                     inp->inp_lport == lport) {
2033                         /*
2034                          * XXX We should be able to directly return
2035                          * the inp here, without any checks.
2036                          * Well unless both bound with SO_REUSEPORT?
2037                          */
2038                         if (prison_flag(inp->inp_cred, PR_IP4))
2039                                 goto found;
2040                         if (tmpinp == NULL)
2041                                 tmpinp = inp;
2042                 }
2043         }
2044         if (tmpinp != NULL) {
2045                 inp = tmpinp;
2046                 goto found;
2047         }
2048
2049 #ifdef  RSS
2050         /*
2051          * For incoming connections, we may wish to do a wildcard
2052          * match for an RSS-local socket.
2053          */
2054         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2055                 struct inpcb *local_wild = NULL, *local_exact = NULL;
2056 #ifdef INET6
2057                 struct inpcb *local_wild_mapped = NULL;
2058 #endif
2059                 struct inpcb *jail_wild = NULL;
2060                 struct inpcbhead *head;
2061                 int injail;
2062
2063                 /*
2064                  * Order of socket selection - we always prefer jails.
2065                  *      1. jailed, non-wild.
2066                  *      2. jailed, wild.
2067                  *      3. non-jailed, non-wild.
2068                  *      4. non-jailed, wild.
2069                  */
2070
2071                 head = &pcbgroup->ipg_hashbase[INP_PCBHASH(INADDR_ANY,
2072                     lport, 0, pcbgroup->ipg_hashmask)];
2073                 CK_LIST_FOREACH(inp, head, inp_pcbgrouphash) {
2074 #ifdef INET6
2075                         /* XXX inp locking */
2076                         if ((inp->inp_vflag & INP_IPV4) == 0)
2077                                 continue;
2078 #endif
2079                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
2080                             inp->inp_lport != lport)
2081                                 continue;
2082
2083                         injail = prison_flag(inp->inp_cred, PR_IP4);
2084                         if (injail) {
2085                                 if (prison_check_ip4(inp->inp_cred,
2086                                     &laddr) != 0)
2087                                         continue;
2088                         } else {
2089                                 if (local_exact != NULL)
2090                                         continue;
2091                         }
2092
2093                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
2094                                 if (injail)
2095                                         goto found;
2096                                 else
2097                                         local_exact = inp;
2098                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
2099 #ifdef INET6
2100                                 /* XXX inp locking, NULL check */
2101                                 if (inp->inp_vflag & INP_IPV6PROTO)
2102                                         local_wild_mapped = inp;
2103                                 else
2104 #endif
2105                                         if (injail)
2106                                                 jail_wild = inp;
2107                                         else
2108                                                 local_wild = inp;
2109                         }
2110                 } /* LIST_FOREACH */
2111
2112                 inp = jail_wild;
2113                 if (inp == NULL)
2114                         inp = local_exact;
2115                 if (inp == NULL)
2116                         inp = local_wild;
2117 #ifdef INET6
2118                 if (inp == NULL)
2119                         inp = local_wild_mapped;
2120 #endif
2121                 if (inp != NULL)
2122                         goto found;
2123         }
2124 #endif
2125
2126         /*
2127          * Then look for a wildcard match, if requested.
2128          */
2129         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2130                 struct inpcb *local_wild = NULL, *local_exact = NULL;
2131 #ifdef INET6
2132                 struct inpcb *local_wild_mapped = NULL;
2133 #endif
2134                 struct inpcb *jail_wild = NULL;
2135                 struct inpcbhead *head;
2136                 int injail;
2137
2138                 /*
2139                  * Order of socket selection - we always prefer jails.
2140                  *      1. jailed, non-wild.
2141                  *      2. jailed, wild.
2142                  *      3. non-jailed, non-wild.
2143                  *      4. non-jailed, wild.
2144                  */
2145                 head = &pcbinfo->ipi_wildbase[INP_PCBHASH(INADDR_ANY, lport,
2146                     0, pcbinfo->ipi_wildmask)];
2147                 CK_LIST_FOREACH(inp, head, inp_pcbgroup_wild) {
2148 #ifdef INET6
2149                         /* XXX inp locking */
2150                         if ((inp->inp_vflag & INP_IPV4) == 0)
2151                                 continue;
2152 #endif
2153                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
2154                             inp->inp_lport != lport)
2155                                 continue;
2156
2157                         injail = prison_flag(inp->inp_cred, PR_IP4);
2158                         if (injail) {
2159                                 if (prison_check_ip4(inp->inp_cred,
2160                                     &laddr) != 0)
2161                                         continue;
2162                         } else {
2163                                 if (local_exact != NULL)
2164                                         continue;
2165                         }
2166
2167                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
2168                                 if (injail)
2169                                         goto found;
2170                                 else
2171                                         local_exact = inp;
2172                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
2173 #ifdef INET6
2174                                 /* XXX inp locking, NULL check */
2175                                 if (inp->inp_vflag & INP_IPV6PROTO)
2176                                         local_wild_mapped = inp;
2177                                 else
2178 #endif
2179                                         if (injail)
2180                                                 jail_wild = inp;
2181                                         else
2182                                                 local_wild = inp;
2183                         }
2184                 } /* LIST_FOREACH */
2185                 inp = jail_wild;
2186                 if (inp == NULL)
2187                         inp = local_exact;
2188                 if (inp == NULL)
2189                         inp = local_wild;
2190 #ifdef INET6
2191                 if (inp == NULL)
2192                         inp = local_wild_mapped;
2193 #endif
2194                 if (inp != NULL)
2195                         goto found;
2196         } /* if (lookupflags & INPLOOKUP_WILDCARD) */
2197         INP_GROUP_UNLOCK(pcbgroup);
2198         return (NULL);
2199
2200 found:
2201         if (lookupflags & INPLOOKUP_WLOCKPCB)
2202                 locked = INP_TRY_WLOCK(inp);
2203         else if (lookupflags & INPLOOKUP_RLOCKPCB)
2204                 locked = INP_TRY_RLOCK(inp);
2205         else
2206                 panic("%s: locking bug", __func__);
2207         if (__predict_false(locked && (inp->inp_flags2 & INP_FREED))) {
2208                 if (lookupflags & INPLOOKUP_WLOCKPCB)
2209                         INP_WUNLOCK(inp);
2210                 else
2211                         INP_RUNLOCK(inp);
2212                 return (NULL);
2213         } else if (!locked)
2214                 in_pcbref(inp);
2215         INP_GROUP_UNLOCK(pcbgroup);
2216         if (!locked) {
2217                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
2218                         INP_WLOCK(inp);
2219                         if (in_pcbrele_wlocked(inp))
2220                                 return (NULL);
2221                 } else {
2222                         INP_RLOCK(inp);
2223                         if (in_pcbrele_rlocked(inp))
2224                                 return (NULL);
2225                 }
2226         }
2227 #ifdef INVARIANTS
2228         if (lookupflags & INPLOOKUP_WLOCKPCB)
2229                 INP_WLOCK_ASSERT(inp);
2230         else
2231                 INP_RLOCK_ASSERT(inp);
2232 #endif
2233         return (inp);
2234 }
2235 #endif /* PCBGROUP */
2236
2237 /*
2238  * Lookup PCB in hash list, using pcbinfo tables.  This variation assumes
2239  * that the caller has locked the hash list, and will not perform any further
2240  * locking or reference operations on either the hash list or the connection.
2241  */
2242 static struct inpcb *
2243 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2244     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
2245     struct ifnet *ifp)
2246 {
2247         struct inpcbhead *head;
2248         struct inpcb *inp, *tmpinp;
2249         u_short fport = fport_arg, lport = lport_arg;
2250
2251         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
2252             ("%s: invalid lookup flags %d", __func__, lookupflags));
2253         INP_HASH_LOCK_ASSERT(pcbinfo);
2254
2255         /*
2256          * First look for an exact match.
2257          */
2258         tmpinp = NULL;
2259         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(faddr.s_addr, lport, fport,
2260             pcbinfo->ipi_hashmask)];
2261         CK_LIST_FOREACH(inp, head, inp_hash) {
2262 #ifdef INET6
2263                 /* XXX inp locking */
2264                 if ((inp->inp_vflag & INP_IPV4) == 0)
2265                         continue;
2266 #endif
2267                 if (inp->inp_faddr.s_addr == faddr.s_addr &&
2268                     inp->inp_laddr.s_addr == laddr.s_addr &&
2269                     inp->inp_fport == fport &&
2270                     inp->inp_lport == lport) {
2271                         /*
2272                          * XXX We should be able to directly return
2273                          * the inp here, without any checks.
2274                          * Well unless both bound with SO_REUSEPORT?
2275                          */
2276                         if (prison_flag(inp->inp_cred, PR_IP4))
2277                                 return (inp);
2278                         if (tmpinp == NULL)
2279                                 tmpinp = inp;
2280                 }
2281         }
2282         if (tmpinp != NULL)
2283                 return (tmpinp);
2284
2285         /*
2286          * Then look in lb group (for wildcard match).
2287          */
2288         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2289                 inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr,
2290                     fport, lookupflags);
2291                 if (inp != NULL)
2292                         return (inp);
2293         }
2294
2295         /*
2296          * Then look for a wildcard match, if requested.
2297          */
2298         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
2299                 struct inpcb *local_wild = NULL, *local_exact = NULL;
2300 #ifdef INET6
2301                 struct inpcb *local_wild_mapped = NULL;
2302 #endif
2303                 struct inpcb *jail_wild = NULL;
2304                 int injail;
2305
2306                 /*
2307                  * Order of socket selection - we always prefer jails.
2308                  *      1. jailed, non-wild.
2309                  *      2. jailed, wild.
2310                  *      3. non-jailed, non-wild.
2311                  *      4. non-jailed, wild.
2312                  */
2313
2314                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(INADDR_ANY, lport,
2315                     0, pcbinfo->ipi_hashmask)];
2316                 CK_LIST_FOREACH(inp, head, inp_hash) {
2317 #ifdef INET6
2318                         /* XXX inp locking */
2319                         if ((inp->inp_vflag & INP_IPV4) == 0)
2320                                 continue;
2321 #endif
2322                         if (inp->inp_faddr.s_addr != INADDR_ANY ||
2323                             inp->inp_lport != lport)
2324                                 continue;
2325
2326                         injail = prison_flag(inp->inp_cred, PR_IP4);
2327                         if (injail) {
2328                                 if (prison_check_ip4(inp->inp_cred,
2329                                     &laddr) != 0)
2330                                         continue;
2331                         } else {
2332                                 if (local_exact != NULL)
2333                                         continue;
2334                         }
2335
2336                         if (inp->inp_laddr.s_addr == laddr.s_addr) {
2337                                 if (injail)
2338                                         return (inp);
2339                                 else
2340                                         local_exact = inp;
2341                         } else if (inp->inp_laddr.s_addr == INADDR_ANY) {
2342 #ifdef INET6
2343                                 /* XXX inp locking, NULL check */
2344                                 if (inp->inp_vflag & INP_IPV6PROTO)
2345                                         local_wild_mapped = inp;
2346                                 else
2347 #endif
2348                                         if (injail)
2349                                                 jail_wild = inp;
2350                                         else
2351                                                 local_wild = inp;
2352                         }
2353                 } /* LIST_FOREACH */
2354                 if (jail_wild != NULL)
2355                         return (jail_wild);
2356                 if (local_exact != NULL)
2357                         return (local_exact);
2358                 if (local_wild != NULL)
2359                         return (local_wild);
2360 #ifdef INET6
2361                 if (local_wild_mapped != NULL)
2362                         return (local_wild_mapped);
2363 #endif
2364         } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
2365
2366         return (NULL);
2367 }
2368
2369 /*
2370  * Lookup PCB in hash list, using pcbinfo tables.  This variation locks the
2371  * hash list lock, and will return the inpcb locked (i.e., requires
2372  * INPLOOKUP_LOCKPCB).
2373  */
2374 static struct inpcb *
2375 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2376     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2377     struct ifnet *ifp)
2378 {
2379         struct inpcb *inp;
2380
2381         inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
2382             (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
2383         if (inp != NULL) {
2384                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
2385                         INP_WLOCK(inp);
2386                         if (__predict_false(inp->inp_flags2 & INP_FREED)) {
2387                                 INP_WUNLOCK(inp);
2388                                 inp = NULL;
2389                         }
2390                 } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
2391                         INP_RLOCK(inp);
2392                         if (__predict_false(inp->inp_flags2 & INP_FREED)) {
2393                                 INP_RUNLOCK(inp);
2394                                 inp = NULL;
2395                         }
2396                 } else
2397                         panic("%s: locking bug", __func__);
2398 #ifdef INVARIANTS
2399                 if (inp != NULL) {
2400                         if (lookupflags & INPLOOKUP_WLOCKPCB)
2401                                 INP_WLOCK_ASSERT(inp);
2402                         else
2403                                 INP_RLOCK_ASSERT(inp);
2404                 }
2405 #endif
2406         }
2407
2408         return (inp);
2409 }
2410
2411 /*
2412  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
2413  * from which a pre-calculated hash value may be extracted.
2414  *
2415  * Possibly more of this logic should be in in_pcbgroup.c.
2416  */
2417 struct inpcb *
2418 in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr faddr, u_int fport,
2419     struct in_addr laddr, u_int lport, int lookupflags, struct ifnet *ifp)
2420 {
2421 #if defined(PCBGROUP) && !defined(RSS)
2422         struct inpcbgroup *pcbgroup;
2423 #endif
2424
2425         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2426             ("%s: invalid lookup flags %d", __func__, lookupflags));
2427         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2428             ("%s: LOCKPCB not set", __func__));
2429
2430         /*
2431          * When not using RSS, use connection groups in preference to the
2432          * reservation table when looking up 4-tuples.  When using RSS, just
2433          * use the reservation table, due to the cost of the Toeplitz hash
2434          * in software.
2435          *
2436          * XXXRW: This policy belongs in the pcbgroup code, as in principle
2437          * we could be doing RSS with a non-Toeplitz hash that is affordable
2438          * in software.
2439          */
2440 #if defined(PCBGROUP) && !defined(RSS)
2441         if (in_pcbgroup_enabled(pcbinfo)) {
2442                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2443                     fport);
2444                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2445                     laddr, lport, lookupflags, ifp));
2446         }
2447 #endif
2448         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2449             lookupflags, ifp));
2450 }
2451
2452 struct inpcb *
2453 in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in_addr faddr,
2454     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
2455     struct ifnet *ifp, struct mbuf *m)
2456 {
2457 #ifdef PCBGROUP
2458         struct inpcbgroup *pcbgroup;
2459 #endif
2460
2461         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
2462             ("%s: invalid lookup flags %d", __func__, lookupflags));
2463         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
2464             ("%s: LOCKPCB not set", __func__));
2465
2466 #ifdef PCBGROUP
2467         /*
2468          * If we can use a hardware-generated hash to look up the connection
2469          * group, use that connection group to find the inpcb.  Otherwise
2470          * fall back on a software hash -- or the reservation table if we're
2471          * using RSS.
2472          *
2473          * XXXRW: As above, that policy belongs in the pcbgroup code.
2474          */
2475         if (in_pcbgroup_enabled(pcbinfo) &&
2476             !(M_HASHTYPE_TEST(m, M_HASHTYPE_NONE))) {
2477                 pcbgroup = in_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
2478                     m->m_pkthdr.flowid);
2479                 if (pcbgroup != NULL)
2480                         return (in_pcblookup_group(pcbinfo, pcbgroup, faddr,
2481                             fport, laddr, lport, lookupflags, ifp));
2482 #ifndef RSS
2483                 pcbgroup = in_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
2484                     fport);
2485                 return (in_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
2486                     laddr, lport, lookupflags, ifp));
2487 #endif
2488         }
2489 #endif
2490         return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2491             lookupflags, ifp));
2492 }
2493 #endif /* INET */
2494
2495 /*
2496  * Insert PCB onto various hash lists.
2497  */
2498 static int
2499 in_pcbinshash_internal(struct inpcb *inp, struct mbuf *m)
2500 {
2501         struct inpcbhead *pcbhash;
2502         struct inpcbporthead *pcbporthash;
2503         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2504         struct inpcbport *phd;
2505         u_int32_t hashkey_faddr;
2506         int so_options;
2507
2508         INP_WLOCK_ASSERT(inp);
2509         INP_HASH_WLOCK_ASSERT(pcbinfo);
2510
2511         KASSERT((inp->inp_flags & INP_INHASHLIST) == 0,
2512             ("in_pcbinshash: INP_INHASHLIST"));
2513
2514 #ifdef INET6
2515         if (inp->inp_vflag & INP_IPV6)
2516                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2517         else
2518 #endif
2519         hashkey_faddr = inp->inp_faddr.s_addr;
2520
2521         pcbhash = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2522                  inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2523
2524         pcbporthash = &pcbinfo->ipi_porthashbase[
2525             INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_porthashmask)];
2526
2527         /*
2528          * Add entry to load balance group.
2529          * Only do this if SO_REUSEPORT_LB is set.
2530          */
2531         so_options = inp_so_options(inp);
2532         if (so_options & SO_REUSEPORT_LB) {
2533                 int ret = in_pcbinslbgrouphash(inp);
2534                 if (ret) {
2535                         /* pcb lb group malloc fail (ret=ENOBUFS). */
2536                         return (ret);
2537                 }
2538         }
2539
2540         /*
2541          * Go through port list and look for a head for this lport.
2542          */
2543         CK_LIST_FOREACH(phd, pcbporthash, phd_hash) {
2544                 if (phd->phd_port == inp->inp_lport)
2545                         break;
2546         }
2547         /*
2548          * If none exists, malloc one and tack it on.
2549          */
2550         if (phd == NULL) {
2551                 phd = malloc(sizeof(struct inpcbport), M_PCB, M_NOWAIT);
2552                 if (phd == NULL) {
2553                         return (ENOBUFS); /* XXX */
2554                 }
2555                 bzero(&phd->phd_epoch_ctx, sizeof(struct epoch_context));
2556                 phd->phd_port = inp->inp_lport;
2557                 CK_LIST_INIT(&phd->phd_pcblist);
2558                 CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2559         }
2560         inp->inp_phd = phd;
2561         CK_LIST_INSERT_HEAD(&phd->phd_pcblist, inp, inp_portlist);
2562         CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
2563         inp->inp_flags |= INP_INHASHLIST;
2564 #ifdef PCBGROUP
2565         if (m != NULL) {
2566                 in_pcbgroup_update_mbuf(inp, m);
2567         } else {
2568                 in_pcbgroup_update(inp);
2569         }
2570 #endif
2571         return (0);
2572 }
2573
2574 int
2575 in_pcbinshash(struct inpcb *inp)
2576 {
2577
2578         return (in_pcbinshash_internal(inp, NULL));
2579 }
2580
2581 int
2582 in_pcbinshash_mbuf(struct inpcb *inp, struct mbuf *m)
2583 {
2584
2585         return (in_pcbinshash_internal(inp, m));
2586 }
2587
2588 /*
2589  * Move PCB to the proper hash bucket when { faddr, fport } have  been
2590  * changed. NOTE: This does not handle the case of the lport changing (the
2591  * hashed port list would have to be updated as well), so the lport must
2592  * not change after in_pcbinshash() has been called.
2593  */
2594 void
2595 in_pcbrehash_mbuf(struct inpcb *inp, struct mbuf *m)
2596 {
2597         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2598         struct inpcbhead *head;
2599         u_int32_t hashkey_faddr;
2600
2601         INP_WLOCK_ASSERT(inp);
2602         INP_HASH_WLOCK_ASSERT(pcbinfo);
2603
2604         KASSERT(inp->inp_flags & INP_INHASHLIST,
2605             ("in_pcbrehash: !INP_INHASHLIST"));
2606
2607 #ifdef INET6
2608         if (inp->inp_vflag & INP_IPV6)
2609                 hashkey_faddr = INP6_PCBHASHKEY(&inp->in6p_faddr);
2610         else
2611 #endif
2612         hashkey_faddr = inp->inp_faddr.s_addr;
2613
2614         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(hashkey_faddr,
2615                 inp->inp_lport, inp->inp_fport, pcbinfo->ipi_hashmask)];
2616
2617         CK_LIST_REMOVE(inp, inp_hash);
2618         CK_LIST_INSERT_HEAD(head, inp, inp_hash);
2619
2620 #ifdef PCBGROUP
2621         if (m != NULL)
2622                 in_pcbgroup_update_mbuf(inp, m);
2623         else
2624                 in_pcbgroup_update(inp);
2625 #endif
2626 }
2627
2628 void
2629 in_pcbrehash(struct inpcb *inp)
2630 {
2631
2632         in_pcbrehash_mbuf(inp, NULL);
2633 }
2634
2635 /*
2636  * Remove PCB from various lists.
2637  */
2638 static void
2639 in_pcbremlists(struct inpcb *inp)
2640 {
2641         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
2642
2643         INP_WLOCK_ASSERT(inp);
2644         INP_LIST_WLOCK_ASSERT(pcbinfo);
2645
2646         inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
2647         if (inp->inp_flags & INP_INHASHLIST) {
2648                 struct inpcbport *phd = inp->inp_phd;
2649
2650                 INP_HASH_WLOCK(pcbinfo);
2651
2652                 /* XXX: Only do if SO_REUSEPORT_LB set? */
2653                 in_pcbremlbgrouphash(inp);
2654
2655                 CK_LIST_REMOVE(inp, inp_hash);
2656                 CK_LIST_REMOVE(inp, inp_portlist);
2657                 if (CK_LIST_FIRST(&phd->phd_pcblist) == NULL) {
2658                         CK_LIST_REMOVE(phd, phd_hash);
2659                         NET_EPOCH_CALL(inpcbport_free, &phd->phd_epoch_ctx);
2660                 }
2661                 INP_HASH_WUNLOCK(pcbinfo);
2662                 inp->inp_flags &= ~INP_INHASHLIST;
2663         }
2664         CK_LIST_REMOVE(inp, inp_list);
2665         pcbinfo->ipi_count--;
2666 #ifdef PCBGROUP
2667         in_pcbgroup_remove(inp);
2668 #endif
2669 }
2670
2671 /*
2672  * Check for alternatives when higher level complains
2673  * about service problems.  For now, invalidate cached
2674  * routing information.  If the route was created dynamically
2675  * (by a redirect), time to try a default gateway again.
2676  */
2677 void
2678 in_losing(struct inpcb *inp)
2679 {
2680
2681         RO_INVALIDATE_CACHE(&inp->inp_route);
2682         return;
2683 }
2684
2685 /*
2686  * A set label operation has occurred at the socket layer, propagate the
2687  * label change into the in_pcb for the socket.
2688  */
2689 void
2690 in_pcbsosetlabel(struct socket *so)
2691 {
2692 #ifdef MAC
2693         struct inpcb *inp;
2694
2695         inp = sotoinpcb(so);
2696         KASSERT(inp != NULL, ("in_pcbsosetlabel: so->so_pcb == NULL"));
2697
2698         INP_WLOCK(inp);
2699         SOCK_LOCK(so);
2700         mac_inpcb_sosetlabel(so, inp);
2701         SOCK_UNLOCK(so);
2702         INP_WUNLOCK(inp);
2703 #endif
2704 }
2705
2706 /*
2707  * ipport_tick runs once per second, determining if random port allocation
2708  * should be continued.  If more than ipport_randomcps ports have been
2709  * allocated in the last second, then we return to sequential port
2710  * allocation. We return to random allocation only once we drop below
2711  * ipport_randomcps for at least ipport_randomtime seconds.
2712  */
2713 static void
2714 ipport_tick(void *xtp)
2715 {
2716         VNET_ITERATOR_DECL(vnet_iter);
2717
2718         VNET_LIST_RLOCK_NOSLEEP();
2719         VNET_FOREACH(vnet_iter) {
2720                 CURVNET_SET(vnet_iter); /* XXX appease INVARIANTS here */
2721                 if (V_ipport_tcpallocs <=
2722                     V_ipport_tcplastcount + V_ipport_randomcps) {
2723                         if (V_ipport_stoprandom > 0)
2724                                 V_ipport_stoprandom--;
2725                 } else
2726                         V_ipport_stoprandom = V_ipport_randomtime;
2727                 V_ipport_tcplastcount = V_ipport_tcpallocs;
2728                 CURVNET_RESTORE();
2729         }
2730         VNET_LIST_RUNLOCK_NOSLEEP();
2731         callout_reset(&ipport_tick_callout, hz, ipport_tick, NULL);
2732 }
2733
2734 static void
2735 ip_fini(void *xtp)
2736 {
2737
2738         callout_stop(&ipport_tick_callout);
2739 }
2740
2741 /*
2742  * The ipport_callout should start running at about the time we attach the
2743  * inet or inet6 domains.
2744  */
2745 static void
2746 ipport_tick_init(const void *unused __unused)
2747 {
2748
2749         /* Start ipport_tick. */
2750         callout_init(&ipport_tick_callout, 1);
2751         callout_reset(&ipport_tick_callout, 1, ipport_tick, NULL);
2752         EVENTHANDLER_REGISTER(shutdown_pre_sync, ip_fini, NULL,
2753                 SHUTDOWN_PRI_DEFAULT);
2754 }
2755 SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
2756     ipport_tick_init, NULL);
2757
2758 void
2759 inp_wlock(struct inpcb *inp)
2760 {
2761
2762         INP_WLOCK(inp);
2763 }
2764
2765 void
2766 inp_wunlock(struct inpcb *inp)
2767 {
2768
2769         INP_WUNLOCK(inp);
2770 }
2771
2772 void
2773 inp_rlock(struct inpcb *inp)
2774 {
2775
2776         INP_RLOCK(inp);
2777 }
2778
2779 void
2780 inp_runlock(struct inpcb *inp)
2781 {
2782
2783         INP_RUNLOCK(inp);
2784 }
2785
2786 #ifdef INVARIANT_SUPPORT
2787 void
2788 inp_lock_assert(struct inpcb *inp)
2789 {
2790
2791         INP_WLOCK_ASSERT(inp);
2792 }
2793
2794 void
2795 inp_unlock_assert(struct inpcb *inp)
2796 {
2797
2798         INP_UNLOCK_ASSERT(inp);
2799 }
2800 #endif
2801
2802 void
2803 inp_apply_all(void (*func)(struct inpcb *, void *), void *arg)
2804 {
2805         struct inpcb *inp;
2806
2807         INP_INFO_WLOCK(&V_tcbinfo);
2808         CK_LIST_FOREACH(inp, V_tcbinfo.ipi_listhead, inp_list) {
2809                 INP_WLOCK(inp);
2810                 func(inp, arg);
2811                 INP_WUNLOCK(inp);
2812         }
2813         INP_INFO_WUNLOCK(&V_tcbinfo);
2814 }
2815
2816 struct socket *
2817 inp_inpcbtosocket(struct inpcb *inp)
2818 {
2819
2820         INP_WLOCK_ASSERT(inp);
2821         return (inp->inp_socket);
2822 }
2823
2824 struct tcpcb *
2825 inp_inpcbtotcpcb(struct inpcb *inp)
2826 {
2827
2828         INP_WLOCK_ASSERT(inp);
2829         return ((struct tcpcb *)inp->inp_ppcb);
2830 }
2831
2832 int
2833 inp_ip_tos_get(const struct inpcb *inp)
2834 {
2835
2836         return (inp->inp_ip_tos);
2837 }
2838
2839 void
2840 inp_ip_tos_set(struct inpcb *inp, int val)
2841 {
2842
2843         inp->inp_ip_tos = val;
2844 }
2845
2846 void
2847 inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
2848     uint32_t *faddr, uint16_t *fp)
2849 {
2850
2851         INP_LOCK_ASSERT(inp);
2852         *laddr = inp->inp_laddr.s_addr;
2853         *faddr = inp->inp_faddr.s_addr;
2854         *lp = inp->inp_lport;
2855         *fp = inp->inp_fport;
2856 }
2857
2858 struct inpcb *
2859 so_sotoinpcb(struct socket *so)
2860 {
2861
2862         return (sotoinpcb(so));
2863 }
2864
2865 struct tcpcb *
2866 so_sototcpcb(struct socket *so)
2867 {
2868
2869         return (sototcpcb(so));
2870 }
2871
2872 /*
2873  * Create an external-format (``xinpcb'') structure using the information in
2874  * the kernel-format in_pcb structure pointed to by inp.  This is done to
2875  * reduce the spew of irrelevant information over this interface, to isolate
2876  * user code from changes in the kernel structure, and potentially to provide
2877  * information-hiding if we decide that some of this information should be
2878  * hidden from users.
2879  */
2880 void
2881 in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
2882 {
2883
2884         bzero(xi, sizeof(*xi));
2885         xi->xi_len = sizeof(struct xinpcb);
2886         if (inp->inp_socket)
2887                 sotoxsocket(inp->inp_socket, &xi->xi_socket);
2888         bcopy(&inp->inp_inc, &xi->inp_inc, sizeof(struct in_conninfo));
2889         xi->inp_gencnt = inp->inp_gencnt;
2890         xi->inp_ppcb = (uintptr_t)inp->inp_ppcb;
2891         xi->inp_flow = inp->inp_flow;
2892         xi->inp_flowid = inp->inp_flowid;
2893         xi->inp_flowtype = inp->inp_flowtype;
2894         xi->inp_flags = inp->inp_flags;
2895         xi->inp_flags2 = inp->inp_flags2;
2896         xi->inp_rss_listen_bucket = inp->inp_rss_listen_bucket;
2897         xi->in6p_cksum = inp->in6p_cksum;
2898         xi->in6p_hops = inp->in6p_hops;
2899         xi->inp_ip_tos = inp->inp_ip_tos;
2900         xi->inp_vflag = inp->inp_vflag;
2901         xi->inp_ip_ttl = inp->inp_ip_ttl;
2902         xi->inp_ip_p = inp->inp_ip_p;
2903         xi->inp_ip_minttl = inp->inp_ip_minttl;
2904 }
2905
2906 #ifdef DDB
2907 static void
2908 db_print_indent(int indent)
2909 {
2910         int i;
2911
2912         for (i = 0; i < indent; i++)
2913                 db_printf(" ");
2914 }
2915
2916 static void
2917 db_print_inconninfo(struct in_conninfo *inc, const char *name, int indent)
2918 {
2919         char faddr_str[48], laddr_str[48];
2920
2921         db_print_indent(indent);
2922         db_printf("%s at %p\n", name, inc);
2923
2924         indent += 2;
2925
2926 #ifdef INET6
2927         if (inc->inc_flags & INC_ISIPV6) {
2928                 /* IPv6. */
2929                 ip6_sprintf(laddr_str, &inc->inc6_laddr);
2930                 ip6_sprintf(faddr_str, &inc->inc6_faddr);
2931         } else
2932 #endif
2933         {
2934                 /* IPv4. */
2935                 inet_ntoa_r(inc->inc_laddr, laddr_str);
2936                 inet_ntoa_r(inc->inc_faddr, faddr_str);
2937         }
2938         db_print_indent(indent);
2939         db_printf("inc_laddr %s   inc_lport %u\n", laddr_str,
2940             ntohs(inc->inc_lport));
2941         db_print_indent(indent);
2942         db_printf("inc_faddr %s   inc_fport %u\n", faddr_str,
2943             ntohs(inc->inc_fport));
2944 }
2945
2946 static void
2947 db_print_inpflags(int inp_flags)
2948 {
2949         int comma;
2950
2951         comma = 0;
2952         if (inp_flags & INP_RECVOPTS) {
2953                 db_printf("%sINP_RECVOPTS", comma ? ", " : "");
2954                 comma = 1;
2955         }
2956         if (inp_flags & INP_RECVRETOPTS) {
2957                 db_printf("%sINP_RECVRETOPTS", comma ? ", " : "");
2958                 comma = 1;
2959         }
2960         if (inp_flags & INP_RECVDSTADDR) {
2961                 db_printf("%sINP_RECVDSTADDR", comma ? ", " : "");
2962                 comma = 1;
2963         }
2964         if (inp_flags & INP_ORIGDSTADDR) {
2965                 db_printf("%sINP_ORIGDSTADDR", comma ? ", " : "");
2966                 comma = 1;
2967         }
2968         if (inp_flags & INP_HDRINCL) {
2969                 db_printf("%sINP_HDRINCL", comma ? ", " : "");
2970                 comma = 1;
2971         }
2972         if (inp_flags & INP_HIGHPORT) {
2973                 db_printf("%sINP_HIGHPORT", comma ? ", " : "");
2974                 comma = 1;
2975         }
2976         if (inp_flags & INP_LOWPORT) {
2977                 db_printf("%sINP_LOWPORT", comma ? ", " : "");
2978                 comma = 1;
2979         }
2980         if (inp_flags & INP_ANONPORT) {
2981                 db_printf("%sINP_ANONPORT", comma ? ", " : "");
2982                 comma = 1;
2983         }
2984         if (inp_flags & INP_RECVIF) {
2985                 db_printf("%sINP_RECVIF", comma ? ", " : "");
2986                 comma = 1;
2987         }
2988         if (inp_flags & INP_MTUDISC) {
2989                 db_printf("%sINP_MTUDISC", comma ? ", " : "");
2990                 comma = 1;
2991         }
2992         if (inp_flags & INP_RECVTTL) {
2993                 db_printf("%sINP_RECVTTL", comma ? ", " : "");
2994                 comma = 1;
2995         }
2996         if (inp_flags & INP_DONTFRAG) {
2997                 db_printf("%sINP_DONTFRAG", comma ? ", " : "");
2998                 comma = 1;
2999         }
3000         if (inp_flags & INP_RECVTOS) {
3001                 db_printf("%sINP_RECVTOS", comma ? ", " : "");
3002                 comma = 1;
3003         }
3004         if (inp_flags & IN6P_IPV6_V6ONLY) {
3005                 db_printf("%sIN6P_IPV6_V6ONLY", comma ? ", " : "");
3006                 comma = 1;
3007         }
3008         if (inp_flags & IN6P_PKTINFO) {
3009                 db_printf("%sIN6P_PKTINFO", comma ? ", " : "");
3010                 comma = 1;
3011         }
3012         if (inp_flags & IN6P_HOPLIMIT) {
3013                 db_printf("%sIN6P_HOPLIMIT", comma ? ", " : "");
3014                 comma = 1;
3015         }
3016         if (inp_flags & IN6P_HOPOPTS) {
3017                 db_printf("%sIN6P_HOPOPTS", comma ? ", " : "");
3018                 comma = 1;
3019         }
3020         if (inp_flags & IN6P_DSTOPTS) {
3021                 db_printf("%sIN6P_DSTOPTS", comma ? ", " : "");
3022                 comma = 1;
3023         }
3024         if (inp_flags & IN6P_RTHDR) {
3025                 db_printf("%sIN6P_RTHDR", comma ? ", " : "");
3026                 comma = 1;
3027         }
3028         if (inp_flags & IN6P_RTHDRDSTOPTS) {
3029                 db_printf("%sIN6P_RTHDRDSTOPTS", comma ? ", " : "");
3030                 comma = 1;
3031         }
3032         if (inp_flags & IN6P_TCLASS) {
3033                 db_printf("%sIN6P_TCLASS", comma ? ", " : "");
3034                 comma = 1;
3035         }
3036         if (inp_flags & IN6P_AUTOFLOWLABEL) {
3037                 db_printf("%sIN6P_AUTOFLOWLABEL", comma ? ", " : "");
3038                 comma = 1;
3039         }
3040         if (inp_flags & INP_TIMEWAIT) {
3041                 db_printf("%sINP_TIMEWAIT", comma ? ", " : "");
3042                 comma  = 1;
3043         }
3044         if (inp_flags & INP_ONESBCAST) {
3045                 db_printf("%sINP_ONESBCAST", comma ? ", " : "");
3046                 comma  = 1;
3047         }
3048         if (inp_flags & INP_DROPPED) {
3049                 db_printf("%sINP_DROPPED", comma ? ", " : "");
3050                 comma  = 1;
3051         }
3052         if (inp_flags & INP_SOCKREF) {
3053                 db_printf("%sINP_SOCKREF", comma ? ", " : "");
3054                 comma  = 1;
3055         }
3056         if (inp_flags & IN6P_RFC2292) {
3057                 db_printf("%sIN6P_RFC2292", comma ? ", " : "");
3058                 comma = 1;
3059         }
3060         if (inp_flags & IN6P_MTU) {
3061                 db_printf("IN6P_MTU%s", comma ? ", " : "");
3062                 comma = 1;
3063         }
3064 }
3065
3066 static void
3067 db_print_inpvflag(u_char inp_vflag)
3068 {
3069         int comma;
3070
3071         comma = 0;
3072         if (inp_vflag & INP_IPV4) {
3073                 db_printf("%sINP_IPV4", comma ? ", " : "");
3074                 comma  = 1;
3075         }
3076         if (inp_vflag & INP_IPV6) {
3077                 db_printf("%sINP_IPV6", comma ? ", " : "");
3078                 comma  = 1;
3079         }
3080         if (inp_vflag & INP_IPV6PROTO) {
3081                 db_printf("%sINP_IPV6PROTO", comma ? ", " : "");
3082                 comma  = 1;
3083         }
3084 }
3085
3086 static void
3087 db_print_inpcb(struct inpcb *inp, const char *name, int indent)
3088 {
3089
3090         db_print_indent(indent);
3091         db_printf("%s at %p\n", name, inp);
3092
3093         indent += 2;
3094
3095         db_print_indent(indent);
3096         db_printf("inp_flow: 0x%x\n", inp->inp_flow);
3097
3098         db_print_inconninfo(&inp->inp_inc, "inp_conninfo", indent);
3099
3100         db_print_indent(indent);
3101         db_printf("inp_ppcb: %p   inp_pcbinfo: %p   inp_socket: %p\n",
3102             inp->inp_ppcb, inp->inp_pcbinfo, inp->inp_socket);
3103
3104         db_print_indent(indent);
3105         db_printf("inp_label: %p   inp_flags: 0x%x (",
3106            inp->inp_label, inp->inp_flags);
3107         db_print_inpflags(inp->inp_flags);
3108         db_printf(")\n");
3109
3110         db_print_indent(indent);
3111         db_printf("inp_sp: %p   inp_vflag: 0x%x (", inp->inp_sp,
3112             inp->inp_vflag);
3113         db_print_inpvflag(inp->inp_vflag);
3114         db_printf(")\n");
3115
3116         db_print_indent(indent);
3117         db_printf("inp_ip_ttl: %d   inp_ip_p: %d   inp_ip_minttl: %d\n",
3118             inp->inp_ip_ttl, inp->inp_ip_p, inp->inp_ip_minttl);
3119
3120         db_print_indent(indent);
3121 #ifdef INET6
3122         if (inp->inp_vflag & INP_IPV6) {
3123                 db_printf("in6p_options: %p   in6p_outputopts: %p   "
3124                     "in6p_moptions: %p\n", inp->in6p_options,
3125                     inp->in6p_outputopts, inp->in6p_moptions);
3126                 db_printf("in6p_icmp6filt: %p   in6p_cksum %d   "
3127                     "in6p_hops %u\n", inp->in6p_icmp6filt, inp->in6p_cksum,
3128                     inp->in6p_hops);
3129         } else
3130 #endif
3131         {
3132                 db_printf("inp_ip_tos: %d   inp_ip_options: %p   "
3133                     "inp_ip_moptions: %p\n", inp->inp_ip_tos,
3134                     inp->inp_options, inp->inp_moptions);
3135         }
3136
3137         db_print_indent(indent);
3138         db_printf("inp_phd: %p   inp_gencnt: %ju\n", inp->inp_phd,
3139             (uintmax_t)inp->inp_gencnt);
3140 }
3141
3142 DB_SHOW_COMMAND(inpcb, db_show_inpcb)
3143 {
3144         struct inpcb *inp;
3145
3146         if (!have_addr) {
3147                 db_printf("usage: show inpcb <addr>\n");
3148                 return;
3149         }
3150         inp = (struct inpcb *)addr;
3151
3152         db_print_inpcb(inp, "inpcb", 0);
3153 }
3154 #endif /* DDB */
3155
3156 #ifdef RATELIMIT
3157 /*
3158  * Modify TX rate limit based on the existing "inp->inp_snd_tag",
3159  * if any.
3160  */
3161 int
3162 in_pcbmodify_txrtlmt(struct inpcb *inp, uint32_t max_pacing_rate)
3163 {
3164         union if_snd_tag_modify_params params = {
3165                 .rate_limit.max_rate = max_pacing_rate,
3166                 .rate_limit.flags = M_NOWAIT,
3167         };
3168         struct m_snd_tag *mst;
3169         struct ifnet *ifp;
3170         int error;
3171
3172         mst = inp->inp_snd_tag;
3173         if (mst == NULL)
3174                 return (EINVAL);
3175
3176         ifp = mst->ifp;
3177         if (ifp == NULL)
3178                 return (EINVAL);
3179
3180         if (ifp->if_snd_tag_modify == NULL) {
3181                 error = EOPNOTSUPP;
3182         } else {
3183                 error = ifp->if_snd_tag_modify(mst, &params);
3184         }
3185         return (error);
3186 }
3187
3188 /*
3189  * Query existing TX rate limit based on the existing
3190  * "inp->inp_snd_tag", if any.
3191  */
3192 int
3193 in_pcbquery_txrtlmt(struct inpcb *inp, uint32_t *p_max_pacing_rate)
3194 {
3195         union if_snd_tag_query_params params = { };
3196         struct m_snd_tag *mst;
3197         struct ifnet *ifp;
3198         int error;
3199
3200         mst = inp->inp_snd_tag;
3201         if (mst == NULL)
3202                 return (EINVAL);
3203
3204         ifp = mst->ifp;
3205         if (ifp == NULL)
3206                 return (EINVAL);
3207
3208         if (ifp->if_snd_tag_query == NULL) {
3209                 error = EOPNOTSUPP;
3210         } else {
3211                 error = ifp->if_snd_tag_query(mst, &params);
3212                 if (error == 0 &&  p_max_pacing_rate != NULL)
3213                         *p_max_pacing_rate = params.rate_limit.max_rate;
3214         }
3215         return (error);
3216 }
3217
3218 /*
3219  * Query existing TX queue level based on the existing
3220  * "inp->inp_snd_tag", if any.
3221  */
3222 int
3223 in_pcbquery_txrlevel(struct inpcb *inp, uint32_t *p_txqueue_level)
3224 {
3225         union if_snd_tag_query_params params = { };
3226         struct m_snd_tag *mst;
3227         struct ifnet *ifp;
3228         int error;
3229
3230         mst = inp->inp_snd_tag;
3231         if (mst == NULL)
3232                 return (EINVAL);
3233
3234         ifp = mst->ifp;
3235         if (ifp == NULL)
3236                 return (EINVAL);
3237
3238         if (ifp->if_snd_tag_query == NULL)
3239                 return (EOPNOTSUPP);
3240
3241         error = ifp->if_snd_tag_query(mst, &params);
3242         if (error == 0 &&  p_txqueue_level != NULL)
3243                 *p_txqueue_level = params.rate_limit.queue_level;
3244         return (error);
3245 }
3246
3247 /*
3248  * Allocate a new TX rate limit send tag from the network interface
3249  * given by the "ifp" argument and save it in "inp->inp_snd_tag":
3250  */
3251 int
3252 in_pcbattach_txrtlmt(struct inpcb *inp, struct ifnet *ifp,
3253     uint32_t flowtype, uint32_t flowid, uint32_t max_pacing_rate, struct m_snd_tag **st)
3254
3255 {
3256         union if_snd_tag_alloc_params params = {
3257                 .rate_limit.hdr.type = (max_pacing_rate == -1U) ?
3258                     IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
3259                 .rate_limit.hdr.flowid = flowid,
3260                 .rate_limit.hdr.flowtype = flowtype,
3261                 .rate_limit.hdr.numa_domain = inp->inp_numa_domain,
3262                 .rate_limit.max_rate = max_pacing_rate,
3263                 .rate_limit.flags = M_NOWAIT,
3264         };
3265         int error;
3266
3267         INP_WLOCK_ASSERT(inp);
3268
3269         if (*st != NULL)
3270                 return (EINVAL);
3271
3272         if (ifp->if_snd_tag_alloc == NULL) {
3273                 error = EOPNOTSUPP;
3274         } else {
3275                 error = ifp->if_snd_tag_alloc(ifp, &params, &inp->inp_snd_tag);
3276
3277 #ifdef INET
3278                 if (error == 0) {
3279                         counter_u64_add(rate_limit_set_ok, 1);
3280                         counter_u64_add(rate_limit_active, 1);
3281                 } else
3282                         counter_u64_add(rate_limit_alloc_fail, 1);
3283 #endif
3284         }
3285         return (error);
3286 }
3287
3288 void
3289 in_pcbdetach_tag(struct ifnet *ifp, struct m_snd_tag *mst)
3290 {
3291         if (ifp == NULL)
3292                 return;
3293
3294         /*
3295          * If the device was detached while we still had reference(s)
3296          * on the ifp, we assume if_snd_tag_free() was replaced with
3297          * stubs.
3298          */
3299         ifp->if_snd_tag_free(mst);
3300
3301         /* release reference count on network interface */
3302         if_rele(ifp);
3303 #ifdef INET
3304         counter_u64_add(rate_limit_active, -1);
3305 #endif
3306 }
3307
3308 /*
3309  * Free an existing TX rate limit tag based on the "inp->inp_snd_tag",
3310  * if any:
3311  */
3312 void
3313 in_pcbdetach_txrtlmt(struct inpcb *inp)
3314 {
3315         struct m_snd_tag *mst;
3316
3317         INP_WLOCK_ASSERT(inp);
3318
3319         mst = inp->inp_snd_tag;
3320         inp->inp_snd_tag = NULL;
3321
3322         if (mst == NULL)
3323                 return;
3324
3325         m_snd_tag_rele(mst);
3326 }
3327
3328 int
3329 in_pcboutput_txrtlmt_locked(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb, uint32_t max_pacing_rate)
3330 {
3331         int error;
3332
3333         /*
3334          * If the existing send tag is for the wrong interface due to
3335          * a route change, first drop the existing tag.  Set the
3336          * CHANGED flag so that we will keep trying to allocate a new
3337          * tag if we fail to allocate one this time.
3338          */
3339         if (inp->inp_snd_tag != NULL && inp->inp_snd_tag->ifp != ifp) {
3340                 in_pcbdetach_txrtlmt(inp);
3341                 inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3342         }
3343
3344         /*
3345          * NOTE: When attaching to a network interface a reference is
3346          * made to ensure the network interface doesn't go away until
3347          * all ratelimit connections are gone. The network interface
3348          * pointers compared below represent valid network interfaces,
3349          * except when comparing towards NULL.
3350          */
3351         if (max_pacing_rate == 0 && inp->inp_snd_tag == NULL) {
3352                 error = 0;
3353         } else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
3354                 if (inp->inp_snd_tag != NULL)
3355                         in_pcbdetach_txrtlmt(inp);
3356                 error = 0;
3357         } else if (inp->inp_snd_tag == NULL) {
3358                 /*
3359                  * In order to utilize packet pacing with RSS, we need
3360                  * to wait until there is a valid RSS hash before we
3361                  * can proceed:
3362                  */
3363                 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
3364                         error = EAGAIN;
3365                 } else {
3366                         error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3367                             mb->m_pkthdr.flowid, max_pacing_rate, &inp->inp_snd_tag);
3368                 }
3369         } else {
3370                 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3371         }
3372         if (error == 0 || error == EOPNOTSUPP)
3373                 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
3374
3375         return (error);
3376 }
3377
3378 /*
3379  * This function should be called when the INP_RATE_LIMIT_CHANGED flag
3380  * is set in the fast path and will attach/detach/modify the TX rate
3381  * limit send tag based on the socket's so_max_pacing_rate value.
3382  */
3383 void
3384 in_pcboutput_txrtlmt(struct inpcb *inp, struct ifnet *ifp, struct mbuf *mb)
3385 {
3386         struct socket *socket;
3387         uint32_t max_pacing_rate;
3388         bool did_upgrade;
3389         int error;
3390
3391         if (inp == NULL)
3392                 return;
3393
3394         socket = inp->inp_socket;
3395         if (socket == NULL)
3396                 return;
3397
3398         if (!INP_WLOCKED(inp)) {
3399                 /*
3400                  * NOTE: If the write locking fails, we need to bail
3401                  * out and use the non-ratelimited ring for the
3402                  * transmit until there is a new chance to get the
3403                  * write lock.
3404                  */
3405                 if (!INP_TRY_UPGRADE(inp))
3406                         return;
3407                 did_upgrade = 1;
3408         } else {
3409                 did_upgrade = 0;
3410         }
3411
3412         /*
3413          * NOTE: The so_max_pacing_rate value is read unlocked,
3414          * because atomic updates are not required since the variable
3415          * is checked at every mbuf we send. It is assumed that the
3416          * variable read itself will be atomic.
3417          */
3418         max_pacing_rate = socket->so_max_pacing_rate;
3419
3420         error = in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3421
3422         if (did_upgrade)
3423                 INP_DOWNGRADE(inp);
3424 }
3425
3426 /*
3427  * Track route changes for TX rate limiting.
3428  */
3429 void
3430 in_pcboutput_eagain(struct inpcb *inp)
3431 {
3432         bool did_upgrade;
3433
3434         if (inp == NULL)
3435                 return;
3436
3437         if (inp->inp_snd_tag == NULL)
3438                 return;
3439
3440         if (!INP_WLOCKED(inp)) {
3441                 /*
3442                  * NOTE: If the write locking fails, we need to bail
3443                  * out and use the non-ratelimited ring for the
3444                  * transmit until there is a new chance to get the
3445                  * write lock.
3446                  */
3447                 if (!INP_TRY_UPGRADE(inp))
3448                         return;
3449                 did_upgrade = 1;
3450         } else {
3451                 did_upgrade = 0;
3452         }
3453
3454         /* detach rate limiting */
3455         in_pcbdetach_txrtlmt(inp);
3456
3457         /* make sure new mbuf send tag allocation is made */
3458         inp->inp_flags2 |= INP_RATE_LIMIT_CHANGED;
3459
3460         if (did_upgrade)
3461                 INP_DOWNGRADE(inp);
3462 }
3463
3464 #ifdef INET
3465 static void
3466 rl_init(void *st)
3467 {
3468         rate_limit_active = counter_u64_alloc(M_WAITOK);
3469         rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK);
3470         rate_limit_set_ok = counter_u64_alloc(M_WAITOK);
3471 }
3472
3473 SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL);
3474 #endif
3475 #endif /* RATELIMIT */