sys/netinet6/in6_pcb.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
   5  * Copyright (c) 2010-2011 Juniper Networks, Inc.
   6  * All rights reserved.
   7  *
   8  * Portions of this software were developed by Robert N. M. Watson under
   9  * contract to Juniper Networks, Inc.
  10  *
  11  * Redistribution and use in source and binary forms, with or without
  12  * modification, are permitted provided that the following conditions
  13  * are met:
  14  * 1. Redistributions of source code must retain the above copyright
  15  *    notice, this list of conditions and the following disclaimer.
  16  * 2. Redistributions in binary form must reproduce the above copyright
  17  *    notice, this list of conditions and the following disclaimer in the
  18  *    documentation and/or other materials provided with the distribution.
  19  * 3. Neither the name of the project nor the names of its contributors
  20  *    may be used to endorse or promote products derived from this software
  21  *    without specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
  24  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  25  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  26  * ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
  27  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  31  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  32  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  33  * SUCH DAMAGE.
  34  *
  35  *      $KAME: in6_pcb.c,v 1.31 2001/05/21 05:45:10 jinmei Exp $
  36  */
  37
  38 /*-
  39  * Copyright (c) 1982, 1986, 1991, 1993
  40  *      The Regents of the University of California.  All rights reserved.
  41  *
  42  * Redistribution and use in source and binary forms, with or without
  43  * modification, are permitted provided that the following conditions
  44  * are met:
  45  * 1. Redistributions of source code must retain the above copyright
  46  *    notice, this list of conditions and the following disclaimer.
  47  * 2. Redistributions in binary form must reproduce the above copyright
  48  *    notice, this list of conditions and the following disclaimer in the
  49  *    documentation and/or other materials provided with the distribution.
  50  * 3. Neither the name of the University nor the names of its contributors
  51  *    may be used to endorse or promote products derived from this software
  52  *    without specific prior written permission.
  53  *
  54  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  55  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  56  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  57  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  58  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  59  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  60  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  61  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  62  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  63  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  64  * SUCH DAMAGE.
  65  *
  66  *      @(#)in_pcb.c    8.2 (Berkeley) 1/4/94
  67  */
  68
  69 #include <sys/cdefs.h>
  70 __FBSDID("$FreeBSD$");
  71
  72 #include "opt_inet.h"
  73 #include "opt_inet6.h"
  74 #include "opt_ipsec.h"
  75 #include "opt_pcbgroup.h"
  76 #include "opt_route.h"
  77 #include "opt_rss.h"
  78
  79 #include <sys/param.h>
  80 #include <sys/systm.h>
  81 #include <sys/malloc.h>
  82 #include <sys/mbuf.h>
  83 #include <sys/domain.h>
  84 #include <sys/protosw.h>
  85 #include <sys/socket.h>
  86 #include <sys/socketvar.h>
  87 #include <sys/sockio.h>
  88 #include <sys/errno.h>
  89 #include <sys/time.h>
  90 #include <sys/priv.h>
  91 #include <sys/proc.h>
  92 #include <sys/jail.h>
  93
  94 #include <vm/uma.h>
  95
  96 #include <net/if.h>
  97 #include <net/if_var.h>
  98 #include <net/if_llatbl.h>
  99 #include <net/if_types.h>
 100 #include <net/route.h>
 101 #include <net/route/nhop.h>
 102
 103 #include <netinet/in.h>
 104 #include <netinet/in_var.h>
 105 #include <netinet/in_systm.h>
 106 #include <netinet/tcp_var.h>
 107 #include <netinet/ip6.h>
 108 #include <netinet/ip_var.h>
 109
 110 #include <netinet6/ip6_var.h>
 111 #include <netinet6/nd6.h>
 112 #include <netinet/in_pcb.h>
 113 #include <netinet6/in6_pcb.h>
 114 #include <netinet6/in6_fib.h>
 115 #include <netinet6/scope6_var.h>
 116
 117 int
 118 in6_pcbbind(struct inpcb *inp, struct sockaddr *nam,
 119     struct ucred *cred)
 120 {
 121         struct socket *so = inp->inp_socket;
 122         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)NULL;
 123         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 124         u_short lport = 0;
 125         int error, lookupflags = 0;
 126         int reuseport = (so->so_options & SO_REUSEPORT);
 127
 128         /*
 129          * XXX: Maybe we could let SO_REUSEPORT_LB set SO_REUSEPORT bit here
 130          * so that we don't have to add to the (already messy) code below.
 131          */
 132         int reuseport_lb = (so->so_options & SO_REUSEPORT_LB);
 133
 134         INP_WLOCK_ASSERT(inp);
 135         INP_HASH_WLOCK_ASSERT(pcbinfo);
 136
 137         if (CK_STAILQ_EMPTY(&V_in6_ifaddrhead)) /* XXX broken! */
 138                 return (EADDRNOTAVAIL);
 139         if (inp->inp_lport || !IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr))
 140                 return (EINVAL);
 141         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
 142                 lookupflags = INPLOOKUP_WILDCARD;
 143         if (nam == NULL) {
 144                 if ((error = prison_local_ip6(cred, &inp->in6p_laddr,
 145                     ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0))) != 0)
 146                         return (error);
 147         } else {
 148                 sin6 = (struct sockaddr_in6 *)nam;
 149                 if (nam->sa_len != sizeof(*sin6))
 150                         return (EINVAL);
 151                 /*
 152                  * family check.
 153                  */
 154                 if (nam->sa_family != AF_INET6)
 155                         return (EAFNOSUPPORT);
 156
 157                 if ((error = sa6_embedscope(sin6, V_ip6_use_defzone)) != 0)
 158                         return(error);
 159
 160                 if ((error = prison_local_ip6(cred, &sin6->sin6_addr,
 161                     ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0))) != 0)
 162                         return (error);
 163
 164                 lport = sin6->sin6_port;
 165                 if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) {
 166                         /*
 167                          * Treat SO_REUSEADDR as SO_REUSEPORT for multicast;
 168                          * allow compepte duplication of binding if
 169                          * SO_REUSEPORT is set, or if SO_REUSEADDR is set
 170                          * and a multicast address is bound on both
 171                          * new and duplicated sockets.
 172                          */
 173                         if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
 174                                 reuseport = SO_REUSEADDR|SO_REUSEPORT;
 175                         /*
 176                          * XXX: How to deal with SO_REUSEPORT_LB here?
 177                          * Treat same as SO_REUSEPORT for now.
 178                          */
 179                         if ((so->so_options &
 180                             (SO_REUSEADDR|SO_REUSEPORT_LB)) != 0)
 181                                 reuseport_lb = SO_REUSEADDR|SO_REUSEPORT_LB;
 182                 } else if (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
 183                         struct epoch_tracker et;
 184                         struct ifaddr *ifa;
 185
 186                         sin6->sin6_port = 0;            /* yech... */
 187                         NET_EPOCH_ENTER(et);
 188                         if ((ifa = ifa_ifwithaddr((struct sockaddr *)sin6)) ==
 189                             NULL &&
 190                             (inp->inp_flags & INP_BINDANY) == 0) {
 191                                 NET_EPOCH_EXIT(et);
 192                                 return (EADDRNOTAVAIL);
 193                         }
 194
 195                         /*
 196                          * XXX: bind to an anycast address might accidentally
 197                          * cause sending a packet with anycast source address.
 198                          * We should allow to bind to a deprecated address, since
 199                          * the application dares to use it.
 200                          */
 201                         if (ifa != NULL &&
 202                             ((struct in6_ifaddr *)ifa)->ia6_flags &
 203                             (IN6_IFF_ANYCAST|IN6_IFF_NOTREADY|IN6_IFF_DETACHED)) {
 204                                 NET_EPOCH_EXIT(et);
 205                                 return (EADDRNOTAVAIL);
 206                         }
 207                         NET_EPOCH_EXIT(et);
 208                 }
 209                 if (lport) {
 210                         struct inpcb *t;
 211                         struct tcptw *tw;
 212
 213                         /* GROSS */
 214                         if (ntohs(lport) <= V_ipport_reservedhigh &&
 215                             ntohs(lport) >= V_ipport_reservedlow &&
 216                             priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
 217                                 return (EACCES);
 218                         if (!IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr) &&
 219                             priv_check_cred(inp->inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
 220                                 t = in6_pcblookup_local(pcbinfo,
 221                                     &sin6->sin6_addr, lport,
 222                                     INPLOOKUP_WILDCARD, cred);
 223                                 if (t &&
 224                                     ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 225                                     ((t->inp_flags & INP_TIMEWAIT) == 0) &&
 226                                     (so->so_type != SOCK_STREAM ||
 227                                      IN6_IS_ADDR_UNSPECIFIED(&t->in6p_faddr)) &&
 228                                     (!IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr) ||
 229                                      !IN6_IS_ADDR_UNSPECIFIED(&t->in6p_laddr) ||
 230                                      (t->inp_flags2 & INP_REUSEPORT) ||
 231                                      (t->inp_flags2 & INP_REUSEPORT_LB) == 0) &&
 232                                     (inp->inp_cred->cr_uid !=
 233                                      t->inp_cred->cr_uid))
 234                                         return (EADDRINUSE);
 235
 236                                 /*
 237                                  * If the socket is a BINDMULTI socket, then
 238                                  * the credentials need to match and the
 239                                  * original socket also has to have been bound
 240                                  * with BINDMULTI.
 241                                  */
 242                                 if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 243                                         return (EADDRINUSE);
 244
 245 #ifdef INET
 246                                 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
 247                                     IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
 248                                         struct sockaddr_in sin;
 249
 250                                         in6_sin6_2_sin(&sin, sin6);
 251                                         t = in_pcblookup_local(pcbinfo,
 252                                             sin.sin_addr, lport,
 253                                             INPLOOKUP_WILDCARD, cred);
 254                                         if (t &&
 255                                             ((inp->inp_flags2 & INP_BINDMULTI) == 0) &&
 256                                             ((t->inp_flags &
 257                                               INP_TIMEWAIT) == 0) &&
 258                                             (so->so_type != SOCK_STREAM ||
 259                                              ntohl(t->inp_faddr.s_addr) ==
 260                                               INADDR_ANY) &&
 261                                             (inp->inp_cred->cr_uid !=
 262                                              t->inp_cred->cr_uid))
 263                                                 return (EADDRINUSE);
 264
 265                                         if (t && (! in_pcbbind_check_bindmulti(inp, t)))
 266                                                 return (EADDRINUSE);
 267                                 }
 268 #endif
 269                         }
 270                         t = in6_pcblookup_local(pcbinfo, &sin6->sin6_addr,
 271                             lport, lookupflags, cred);
 272                         if (t && (t->inp_flags & INP_TIMEWAIT)) {
 273                                 /*
 274                                  * XXXRW: If an incpb has had its timewait
 275                                  * state recycled, we treat the address as
 276                                  * being in use (for now).  This is better
 277                                  * than a panic, but not desirable.
 278                                  */
 279                                 tw = intotw(t);
 280                                 if (tw == NULL ||
 281                                     ((reuseport & tw->tw_so_options) == 0 &&
 282                                          (reuseport_lb & tw->tw_so_options) == 0))
 283                                         return (EADDRINUSE);
 284                         } else if (t && (reuseport & inp_so_options(t)) == 0 &&
 285                                            (reuseport_lb & inp_so_options(t)) == 0) {
 286                                 return (EADDRINUSE);
 287                         }
 288 #ifdef INET
 289                         if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
 290                             IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr)) {
 291                                 struct sockaddr_in sin;
 292
 293                                 in6_sin6_2_sin(&sin, sin6);
 294                                 t = in_pcblookup_local(pcbinfo, sin.sin_addr,
 295                                    lport, lookupflags, cred);
 296                                 if (t && t->inp_flags & INP_TIMEWAIT) {
 297                                         tw = intotw(t);
 298                                         if (tw == NULL)
 299                                                 return (EADDRINUSE);
 300                                         if ((reuseport & tw->tw_so_options) == 0
 301                                             && (reuseport_lb & tw->tw_so_options) == 0
 302                                             && (ntohl(t->inp_laddr.s_addr) !=
 303                                                 INADDR_ANY || ((inp->inp_vflag &
 304                                                         INP_IPV6PROTO) ==
 305                                                     (t->inp_vflag & INP_IPV6PROTO))))
 306                                                 return (EADDRINUSE);
 307                                 } else if (t &&
 308                                     (reuseport & inp_so_options(t)) == 0 &&
 309                                     (reuseport_lb & inp_so_options(t)) == 0 &&
 310                                     (ntohl(t->inp_laddr.s_addr) != INADDR_ANY ||
 311                                         (t->inp_vflag & INP_IPV6PROTO) != 0)) {
 312                                         return (EADDRINUSE);
 313                                 }
 314                         }
 315 #endif
 316                 }
 317                 inp->in6p_laddr = sin6->sin6_addr;
 318         }
 319         if (lport == 0) {
 320                 if ((error = in6_pcbsetport(&inp->in6p_laddr, inp, cred)) != 0) {
 321                         /* Undo an address bind that may have occurred. */
 322                         inp->in6p_laddr = in6addr_any;
 323                         return (error);
 324                 }
 325         } else {
 326                 inp->inp_lport = lport;
 327                 if (in_pcbinshash(inp) != 0) {
 328                         inp->in6p_laddr = in6addr_any;
 329                         inp->inp_lport = 0;
 330                         return (EAGAIN);
 331                 }
 332         }
 333         return (0);
 334 }
 335
 336 /*
 337  *   Transform old in6_pcbconnect() into an inner subroutine for new
 338  *   in6_pcbconnect(): Do some validity-checking on the remote
 339  *   address (in mbuf 'nam') and then determine local host address
 340  *   (i.e., which interface) to use to access that remote host.
 341  *
 342  *   This preserves definition of in6_pcbconnect(), while supporting a
 343  *   slightly different version for T/TCP.  (This is more than
 344  *   a bit of a kludge, but cleaning up the internal interfaces would
 345  *   have forced minor changes in every protocol).
 346  */
 347 static int
 348 in6_pcbladdr(struct inpcb *inp, struct sockaddr *nam,
 349     struct in6_addr *plocal_addr6)
 350 {
 351         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)nam;
 352         int error = 0;
 353         int scope_ambiguous = 0;
 354         struct in6_addr in6a;
 355
 356         INP_WLOCK_ASSERT(inp);
 357         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);        /* XXXRW: why? */
 358
 359         if (nam->sa_len != sizeof (*sin6))
 360                 return (EINVAL);
 361         if (sin6->sin6_family != AF_INET6)
 362                 return (EAFNOSUPPORT);
 363         if (sin6->sin6_port == 0)
 364                 return (EADDRNOTAVAIL);
 365
 366         if (sin6->sin6_scope_id == 0 && !V_ip6_use_defzone)
 367                 scope_ambiguous = 1;
 368         if ((error = sa6_embedscope(sin6, V_ip6_use_defzone)) != 0)
 369                 return(error);
 370
 371         if (!CK_STAILQ_EMPTY(&V_in6_ifaddrhead)) {
 372                 /*
 373                  * If the destination address is UNSPECIFIED addr,
 374                  * use the loopback addr, e.g ::1.
 375                  */
 376                 if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr))
 377                         sin6->sin6_addr = in6addr_loopback;
 378         }
 379         if ((error = prison_remote_ip6(inp->inp_cred, &sin6->sin6_addr)) != 0)
 380                 return (error);
 381
 382         error = in6_selectsrc_socket(sin6, inp->in6p_outputopts,
 383             inp, inp->inp_cred, scope_ambiguous, &in6a, NULL);
 384         if (error)
 385                 return (error);
 386
 387         /*
 388          * Do not update this earlier, in case we return with an error.
 389          *
 390          * XXX: this in6_selectsrc_socket result might replace the bound local
 391          * address with the address specified by setsockopt(IPV6_PKTINFO).
 392          * Is it the intended behavior?
 393          */
 394         *plocal_addr6 = in6a;
 395
 396         /*
 397          * Don't do pcblookup call here; return interface in
 398          * plocal_addr6
 399          * and exit to caller, that will do the lookup.
 400          */
 401
 402         return (0);
 403 }
 404
 405 /*
 406  * Outer subroutine:
 407  * Connect from a socket to a specified address.
 408  * Both address and port must be specified in argument sin.
 409  * If don't have a local address for this socket yet,
 410  * then pick one.
 411  */
 412 int
 413 in6_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr *nam,
 414     struct ucred *cred, struct mbuf *m, bool rehash)
 415 {
 416         struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 417         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)nam;
 418         struct sockaddr_in6 laddr6;
 419         int error;
 420
 421         bzero(&laddr6, sizeof(laddr6));
 422         laddr6.sin6_family = AF_INET6;
 423
 424         INP_WLOCK_ASSERT(inp);
 425         INP_HASH_WLOCK_ASSERT(pcbinfo);
 426
 427 #ifdef ROUTE_MPATH
 428         if (CALC_FLOWID_OUTBOUND) {
 429                 uint32_t hash_type, hash_val;
 430
 431                 hash_val = fib6_calc_software_hash(&inp->in6p_laddr,
 432                     &sin6->sin6_addr, 0, sin6->sin6_port,
 433                     inp->inp_socket->so_proto->pr_protocol, &hash_type);
 434                 inp->inp_flowid = hash_val;
 435                 inp->inp_flowtype = hash_type;
 436         }
 437 #endif
 438         /*
 439          * Call inner routine, to assign local interface address.
 440          * in6_pcbladdr() may automatically fill in sin6_scope_id.
 441          */
 442         if ((error = in6_pcbladdr(inp, nam, &laddr6.sin6_addr)) != 0)
 443                 return (error);
 444
 445         if (in6_pcblookup_hash_locked(pcbinfo, &sin6->sin6_addr,
 446                                sin6->sin6_port,
 447                               IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)
 448                               ? &laddr6.sin6_addr : &inp->in6p_laddr,
 449                               inp->inp_lport, 0, NULL, M_NODOM) != NULL) {
 450                 return (EADDRINUSE);
 451         }
 452         if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
 453                 if (inp->inp_lport == 0) {
 454                         /*
 455                          * rehash was required to be true in the past for
 456                          * this case; retain that convention.  However,
 457                          * we now call in_pcb_lport_dest rather than
 458                          * in6_pcbbind; the former does not insert into
 459                          * the hash table, the latter does.  Change rehash
 460                          * to false to do the in_pcbinshash below.
 461                          */
 462                         KASSERT(rehash == true,
 463                             ("Rehashing required for unbound inps"));
 464                         rehash = false;
 465                         error = in_pcb_lport_dest(inp,
 466                             (struct sockaddr *) &laddr6, &inp->inp_lport,
 467                             (struct sockaddr *) sin6, sin6->sin6_port, cred,
 468                             INPLOOKUP_WILDCARD);
 469                         if (error)
 470                                 return (error);
 471                 }
 472                 inp->in6p_laddr = laddr6.sin6_addr;
 473         }
 474         inp->in6p_faddr = sin6->sin6_addr;
 475         inp->inp_fport = sin6->sin6_port;
 476         /* update flowinfo - draft-itojun-ipv6-flowlabel-api-00 */
 477         inp->inp_flow &= ~IPV6_FLOWLABEL_MASK;
 478         if (inp->inp_flags & IN6P_AUTOFLOWLABEL)
 479                 inp->inp_flow |=
 480                     (htonl(ip6_randomflowlabel()) & IPV6_FLOWLABEL_MASK);
 481
 482         if (rehash) {
 483                 in_pcbrehash_mbuf(inp, m);
 484         } else {
 485                 in_pcbinshash_mbuf(inp, m);
 486         }
 487
 488         return (0);
 489 }
 490
 491 int
 492 in6_pcbconnect(struct inpcb *inp, struct sockaddr *nam, struct ucred *cred)
 493 {
 494
 495         return (in6_pcbconnect_mbuf(inp, nam, cred, NULL, true));
 496 }
 497
 498 void
 499 in6_pcbdisconnect(struct inpcb *inp)
 500 {
 501
 502         INP_WLOCK_ASSERT(inp);
 503         INP_HASH_WLOCK_ASSERT(inp->inp_pcbinfo);
 504
 505         bzero((caddr_t)&inp->in6p_faddr, sizeof(inp->in6p_faddr));
 506         inp->inp_fport = 0;
 507         /* clear flowinfo - draft-itojun-ipv6-flowlabel-api-00 */
 508         inp->inp_flow &= ~IPV6_FLOWLABEL_MASK;
 509         in_pcbrehash(inp);
 510 }
 511
 512 struct sockaddr *
 513 in6_sockaddr(in_port_t port, struct in6_addr *addr_p)
 514 {
 515         struct sockaddr_in6 *sin6;
 516
 517         sin6 = malloc(sizeof *sin6, M_SONAME, M_WAITOK);
 518         bzero(sin6, sizeof *sin6);
 519         sin6->sin6_family = AF_INET6;
 520         sin6->sin6_len = sizeof(*sin6);
 521         sin6->sin6_port = port;
 522         sin6->sin6_addr = *addr_p;
 523         (void)sa6_recoverscope(sin6); /* XXX: should catch errors */
 524
 525         return (struct sockaddr *)sin6;
 526 }
 527
 528 struct sockaddr *
 529 in6_v4mapsin6_sockaddr(in_port_t port, struct in_addr *addr_p)
 530 {
 531         struct sockaddr_in sin;
 532         struct sockaddr_in6 *sin6_p;
 533
 534         bzero(&sin, sizeof sin);
 535         sin.sin_family = AF_INET;
 536         sin.sin_len = sizeof(sin);
 537         sin.sin_port = port;
 538         sin.sin_addr = *addr_p;
 539
 540         sin6_p = malloc(sizeof *sin6_p, M_SONAME,
 541                 M_WAITOK);
 542         in6_sin_2_v4mapsin6(&sin, sin6_p);
 543
 544         return (struct sockaddr *)sin6_p;
 545 }
 546
 547 int
 548 in6_getsockaddr(struct socket *so, struct sockaddr **nam)
 549 {
 550         struct inpcb *inp;
 551         struct in6_addr addr;
 552         in_port_t port;
 553
 554         inp = sotoinpcb(so);
 555         KASSERT(inp != NULL, ("in6_getsockaddr: inp == NULL"));
 556
 557         INP_RLOCK(inp);
 558         port = inp->inp_lport;
 559         addr = inp->in6p_laddr;
 560         INP_RUNLOCK(inp);
 561
 562         *nam = in6_sockaddr(port, &addr);
 563         return 0;
 564 }
 565
 566 int
 567 in6_getpeeraddr(struct socket *so, struct sockaddr **nam)
 568 {
 569         struct inpcb *inp;
 570         struct in6_addr addr;
 571         in_port_t port;
 572
 573         inp = sotoinpcb(so);
 574         KASSERT(inp != NULL, ("in6_getpeeraddr: inp == NULL"));
 575
 576         INP_RLOCK(inp);
 577         port = inp->inp_fport;
 578         addr = inp->in6p_faddr;
 579         INP_RUNLOCK(inp);
 580
 581         *nam = in6_sockaddr(port, &addr);
 582         return 0;
 583 }
 584
 585 int
 586 in6_mapped_sockaddr(struct socket *so, struct sockaddr **nam)
 587 {
 588         struct  inpcb *inp;
 589         int     error;
 590
 591         inp = sotoinpcb(so);
 592         KASSERT(inp != NULL, ("in6_mapped_sockaddr: inp == NULL"));
 593
 594 #ifdef INET
 595         if ((inp->inp_vflag & (INP_IPV4 | INP_IPV6)) == INP_IPV4) {
 596                 error = in_getsockaddr(so, nam);
 597                 if (error == 0)
 598                         in6_sin_2_v4mapsin6_in_sock(nam);
 599         } else
 600 #endif
 601         {
 602                 /* scope issues will be handled in in6_getsockaddr(). */
 603                 error = in6_getsockaddr(so, nam);
 604         }
 605
 606         return error;
 607 }
 608
 609 int
 610 in6_mapped_peeraddr(struct socket *so, struct sockaddr **nam)
 611 {
 612         struct  inpcb *inp;
 613         int     error;
 614
 615         inp = sotoinpcb(so);
 616         KASSERT(inp != NULL, ("in6_mapped_peeraddr: inp == NULL"));
 617
 618 #ifdef INET
 619         if ((inp->inp_vflag & (INP_IPV4 | INP_IPV6)) == INP_IPV4) {
 620                 error = in_getpeeraddr(so, nam);
 621                 if (error == 0)
 622                         in6_sin_2_v4mapsin6_in_sock(nam);
 623         } else
 624 #endif
 625         /* scope issues will be handled in in6_getpeeraddr(). */
 626         error = in6_getpeeraddr(so, nam);
 627
 628         return error;
 629 }
 630
 631 /*
 632  * Pass some notification to all connections of a protocol
 633  * associated with address dst.  The local address and/or port numbers
 634  * may be specified to limit the search.  The "usual action" will be
 635  * taken, depending on the ctlinput cmd.  The caller must filter any
 636  * cmds that are uninteresting (e.g., no error in the map).
 637  * Call the protocol specific routine (if any) to report
 638  * any errors for each matching socket.
 639  */
 640 void
 641 in6_pcbnotify(struct inpcbinfo *pcbinfo, struct sockaddr *dst,
 642     u_int fport_arg, const struct sockaddr *src, u_int lport_arg,
 643     int cmd, void *cmdarg,
 644     struct inpcb *(*notify)(struct inpcb *, int))
 645 {
 646         struct inpcb *inp, *inp_temp;
 647         struct sockaddr_in6 sa6_src, *sa6_dst;
 648         u_short fport = fport_arg, lport = lport_arg;
 649         u_int32_t flowinfo;
 650         int errno;
 651
 652         if ((unsigned)cmd >= PRC_NCMDS || dst->sa_family != AF_INET6)
 653                 return;
 654
 655         sa6_dst = (struct sockaddr_in6 *)dst;
 656         if (IN6_IS_ADDR_UNSPECIFIED(&sa6_dst->sin6_addr))
 657                 return;
 658
 659         /*
 660          * note that src can be NULL when we get notify by local fragmentation.
 661          */
 662         sa6_src = (src == NULL) ? sa6_any : *(const struct sockaddr_in6 *)src;
 663         flowinfo = sa6_src.sin6_flowinfo;
 664
 665         /*
 666          * Redirects go to all references to the destination,
 667          * and use in6_rtchange to invalidate the route cache.
 668          * Dead host indications: also use in6_rtchange to invalidate
 669          * the cache, and deliver the error to all the sockets.
 670          * Otherwise, if we have knowledge of the local port and address,
 671          * deliver only to that socket.
 672          */
 673         if (PRC_IS_REDIRECT(cmd) || cmd == PRC_HOSTDEAD) {
 674                 fport = 0;
 675                 lport = 0;
 676                 bzero((caddr_t)&sa6_src.sin6_addr, sizeof(sa6_src.sin6_addr));
 677
 678                 if (cmd != PRC_HOSTDEAD)
 679                         notify = in6_rtchange;
 680         }
 681         errno = inet6ctlerrmap[cmd];
 682         INP_INFO_WLOCK(pcbinfo);
 683         CK_LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, inp_temp) {
 684                 INP_WLOCK(inp);
 685                 if ((inp->inp_vflag & INP_IPV6) == 0) {
 686                         INP_WUNLOCK(inp);
 687                         continue;
 688                 }
 689
 690                 /*
 691                  * If the error designates a new path MTU for a destination
 692                  * and the application (associated with this socket) wanted to
 693                  * know the value, notify.
 694                  * XXX: should we avoid to notify the value to TCP sockets?
 695                  */
 696                 if (cmd == PRC_MSGSIZE && cmdarg != NULL)
 697                         ip6_notify_pmtu(inp, (struct sockaddr_in6 *)dst,
 698                                         *(u_int32_t *)cmdarg);
 699
 700                 /*
 701                  * Detect if we should notify the error. If no source and
 702                  * destination ports are specifed, but non-zero flowinfo and
 703                  * local address match, notify the error. This is the case
 704                  * when the error is delivered with an encrypted buffer
 705                  * by ESP. Otherwise, just compare addresses and ports
 706                  * as usual.
 707                  */
 708                 if (lport == 0 && fport == 0 && flowinfo &&
 709                     inp->inp_socket != NULL &&
 710                     flowinfo == (inp->inp_flow & IPV6_FLOWLABEL_MASK) &&
 711                     IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, &sa6_src.sin6_addr))
 712                         goto do_notify;
 713                 else if (!IN6_ARE_ADDR_EQUAL(&inp->in6p_faddr,
 714                                              &sa6_dst->sin6_addr) ||
 715                          inp->inp_socket == 0 ||
 716                          (lport && inp->inp_lport != lport) ||
 717                          (!IN6_IS_ADDR_UNSPECIFIED(&sa6_src.sin6_addr) &&
 718                           !IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr,
 719                                               &sa6_src.sin6_addr)) ||
 720                          (fport && inp->inp_fport != fport)) {
 721                         INP_WUNLOCK(inp);
 722                         continue;
 723                 }
 724
 725           do_notify:
 726                 if (notify) {
 727                         if ((*notify)(inp, errno))
 728                                 INP_WUNLOCK(inp);
 729                 } else
 730                         INP_WUNLOCK(inp);
 731         }
 732         INP_INFO_WUNLOCK(pcbinfo);
 733 }
 734
 735 /*
 736  * Lookup a PCB based on the local address and port.  Caller must hold the
 737  * hash lock.  No inpcb locks or references are acquired.
 738  */
 739 struct inpcb *
 740 in6_pcblookup_local(struct inpcbinfo *pcbinfo, struct in6_addr *laddr,
 741     u_short lport, int lookupflags, struct ucred *cred)
 742 {
 743         struct inpcb *inp;
 744         int matchwild = 3, wildcard;
 745
 746         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
 747             ("%s: invalid lookup flags %d", __func__, lookupflags));
 748
 749         INP_HASH_LOCK_ASSERT(pcbinfo);
 750
 751         if ((lookupflags & INPLOOKUP_WILDCARD) == 0) {
 752                 struct inpcbhead *head;
 753                 /*
 754                  * Look for an unconnected (wildcard foreign addr) PCB that
 755                  * matches the local address and port we're looking for.
 756                  */
 757                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(
 758                     INP6_PCBHASHKEY(&in6addr_any), lport, 0,
 759                     pcbinfo->ipi_hashmask)];
 760                 CK_LIST_FOREACH(inp, head, inp_hash) {
 761                         /* XXX inp locking */
 762                         if ((inp->inp_vflag & INP_IPV6) == 0)
 763                                 continue;
 764                         if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) &&
 765                             IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr) &&
 766                             inp->inp_lport == lport) {
 767                                 /* Found. */
 768                                 if (cred == NULL ||
 769                                     prison_equal_ip6(cred->cr_prison,
 770                                         inp->inp_cred->cr_prison))
 771                                         return (inp);
 772                         }
 773                 }
 774                 /*
 775                  * Not found.
 776                  */
 777                 return (NULL);
 778         } else {
 779                 struct inpcbporthead *porthash;
 780                 struct inpcbport *phd;
 781                 struct inpcb *match = NULL;
 782                 /*
 783                  * Best fit PCB lookup.
 784                  *
 785                  * First see if this local port is in use by looking on the
 786                  * port hash list.
 787                  */
 788                 porthash = &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
 789                     pcbinfo->ipi_porthashmask)];
 790                 CK_LIST_FOREACH(phd, porthash, phd_hash) {
 791                         if (phd->phd_port == lport)
 792                                 break;
 793                 }
 794                 if (phd != NULL) {
 795                         /*
 796                          * Port is in use by one or more PCBs. Look for best
 797                          * fit.
 798                          */
 799                         CK_LIST_FOREACH(inp, &phd->phd_pcblist, inp_portlist) {
 800                                 wildcard = 0;
 801                                 if (cred != NULL &&
 802                                     !prison_equal_ip6(cred->cr_prison,
 803                                         inp->inp_cred->cr_prison))
 804                                         continue;
 805                                 /* XXX inp locking */
 806                                 if ((inp->inp_vflag & INP_IPV6) == 0)
 807                                         continue;
 808                                 if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr))
 809                                         wildcard++;
 810                                 if (!IN6_IS_ADDR_UNSPECIFIED(
 811                                         &inp->in6p_laddr)) {
 812                                         if (IN6_IS_ADDR_UNSPECIFIED(laddr))
 813                                                 wildcard++;
 814                                         else if (!IN6_ARE_ADDR_EQUAL(
 815                                             &inp->in6p_laddr, laddr))
 816                                                 continue;
 817                                 } else {
 818                                         if (!IN6_IS_ADDR_UNSPECIFIED(laddr))
 819                                                 wildcard++;
 820                                 }
 821                                 if (wildcard < matchwild) {
 822                                         match = inp;
 823                                         matchwild = wildcard;
 824                                         if (matchwild == 0)
 825                                                 break;
 826                                 }
 827                         }
 828                 }
 829                 return (match);
 830         }
 831 }
 832
 833 void
 834 in6_pcbpurgeif0(struct inpcbinfo *pcbinfo, struct ifnet *ifp)
 835 {
 836         struct inpcb *inp;
 837         struct in6_multi *inm;
 838         struct in6_mfilter *imf;
 839         struct ip6_moptions *im6o;
 840
 841         INP_INFO_WLOCK(pcbinfo);
 842         CK_LIST_FOREACH(inp, pcbinfo->ipi_listhead, inp_list) {
 843                 INP_WLOCK(inp);
 844                 if (__predict_false(inp->inp_flags2 & INP_FREED)) {
 845                         INP_WUNLOCK(inp);
 846                         continue;
 847                 }
 848                 im6o = inp->in6p_moptions;
 849                 if ((inp->inp_vflag & INP_IPV6) && im6o != NULL) {
 850                         /*
 851                          * Unselect the outgoing ifp for multicast if it
 852                          * is being detached.
 853                          */
 854                         if (im6o->im6o_multicast_ifp == ifp)
 855                                 im6o->im6o_multicast_ifp = NULL;
 856                         /*
 857                          * Drop multicast group membership if we joined
 858                          * through the interface being detached.
 859                          */
 860 restart:
 861                         IP6_MFILTER_FOREACH(imf, &im6o->im6o_head) {
 862                                 if ((inm = imf->im6f_in6m) == NULL)
 863                                         continue;
 864                                 if (inm->in6m_ifp != ifp)
 865                                         continue;
 866                                 ip6_mfilter_remove(&im6o->im6o_head, imf);
 867                                 IN6_MULTI_LOCK_ASSERT();
 868                                 in6_leavegroup_locked(inm, NULL);
 869                                 ip6_mfilter_free(imf);
 870                                 goto restart;
 871                         }
 872                 }
 873                 INP_WUNLOCK(inp);
 874         }
 875         INP_INFO_WUNLOCK(pcbinfo);
 876 }
 877
 878 /*
 879  * Check for alternatives when higher level complains
 880  * about service problems.  For now, invalidate cached
 881  * routing information.  If the route was created dynamically
 882  * (by a redirect), time to try a default gateway again.
 883  */
 884 void
 885 in6_losing(struct inpcb *inp)
 886 {
 887
 888         RO_INVALIDATE_CACHE(&inp->inp_route6);
 889 }
 890
 891 /*
 892  * After a routing change, flush old routing
 893  * and allocate a (hopefully) better one.
 894  */
 895 struct inpcb *
 896 in6_rtchange(struct inpcb *inp, int errno __unused)
 897 {
 898
 899         RO_INVALIDATE_CACHE(&inp->inp_route6);
 900         return inp;
 901 }
 902
 903 static struct inpcb *
 904 in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
 905     const struct in6_addr *laddr, uint16_t lport, const struct in6_addr *faddr,
 906     uint16_t fport, int lookupflags, uint8_t numa_domain)
 907 {
 908         struct inpcb *local_wild, *numa_wild;
 909         const struct inpcblbgrouphead *hdr;
 910         struct inpcblbgroup *grp;
 911         uint32_t idx;
 912
 913         INP_HASH_LOCK_ASSERT(pcbinfo);
 914
 915         hdr = &pcbinfo->ipi_lbgrouphashbase[
 916             INP_PCBPORTHASH(lport, pcbinfo->ipi_lbgrouphashmask)];
 917
 918         /*
 919          * Order of socket selection:
 920          * 1. non-wild.
 921          * 2. wild (if lookupflags contains INPLOOKUP_WILDCARD).
 922          *
 923          * NOTE:
 924          * - Load balanced group does not contain jailed sockets.
 925          * - Load balanced does not contain IPv4 mapped INET6 wild sockets.
 926          */
 927         local_wild = NULL;
 928         numa_wild = NULL;
 929         CK_LIST_FOREACH(grp, hdr, il_list) {
 930 #ifdef INET
 931                 if (!(grp->il_vflag & INP_IPV6))
 932                         continue;
 933 #endif
 934                 if (grp->il_lport != lport)
 935                         continue;
 936
 937                 idx = INP_PCBLBGROUP_PKTHASH(INP6_PCBHASHKEY(faddr), lport,
 938                     fport) % grp->il_inpcnt;
 939                 if (IN6_ARE_ADDR_EQUAL(&grp->il6_laddr, laddr)) {
 940                         if (numa_domain == M_NODOM ||
 941                             grp->il_numa_domain == numa_domain) {
 942                                 return (grp->il_inp[idx]);
 943                         }
 944                         else
 945                                 numa_wild = grp->il_inp[idx];
 946                 }
 947                 if (IN6_IS_ADDR_UNSPECIFIED(&grp->il6_laddr) &&
 948                     (lookupflags & INPLOOKUP_WILDCARD) != 0 &&
 949                     (local_wild == NULL || numa_domain == M_NODOM ||
 950                         grp->il_numa_domain == numa_domain)) {
 951                         local_wild = grp->il_inp[idx];
 952                 }
 953         }
 954         if (numa_wild != NULL)
 955                 return (numa_wild);
 956         return (local_wild);
 957 }
 958
 959 #ifdef PCBGROUP
 960 /*
 961  * Lookup PCB in hash list, using pcbgroup tables.
 962  */
 963 static struct inpcb *
 964 in6_pcblookup_group(struct inpcbinfo *pcbinfo, struct inpcbgroup *pcbgroup,
 965     struct in6_addr *faddr, u_int fport_arg, struct in6_addr *laddr,
 966     u_int lport_arg, int lookupflags, struct ifnet *ifp)
 967 {
 968         struct inpcbhead *head;
 969         struct inpcb *inp, *tmpinp;
 970         u_short fport = fport_arg, lport = lport_arg;
 971         bool locked;
 972
 973         /*
 974          * First look for an exact match.
 975          */
 976         tmpinp = NULL;
 977         INP_GROUP_LOCK(pcbgroup);
 978         head = &pcbgroup->ipg_hashbase[INP_PCBHASH(
 979             INP6_PCBHASHKEY(faddr), lport, fport, pcbgroup->ipg_hashmask)];
 980         CK_LIST_FOREACH(inp, head, inp_pcbgrouphash) {
 981                 /* XXX inp locking */
 982                 if ((inp->inp_vflag & INP_IPV6) == 0)
 983                         continue;
 984                 if (IN6_ARE_ADDR_EQUAL(&inp->in6p_faddr, faddr) &&
 985                     IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr) &&
 986                     inp->inp_fport == fport &&
 987                     inp->inp_lport == lport) {
 988                         /*
 989                          * XXX We should be able to directly return
 990                          * the inp here, without any checks.
 991                          * Well unless both bound with SO_REUSEPORT?
 992                          */
 993                         if (prison_flag(inp->inp_cred, PR_IP6))
 994                                 goto found;
 995                         if (tmpinp == NULL)
 996                                 tmpinp = inp;
 997                 }
 998         }
 999         if (tmpinp != NULL) {
1000                 inp = tmpinp;
1001                 goto found;
1002         }
1003
1004         /*
1005          * Then look for a wildcard match in the pcbgroup.
1006          */
1007         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1008                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1009                 struct inpcb *jail_wild = NULL;
1010                 int injail;
1011
1012                 /*
1013                  * Order of socket selection - we always prefer jails.
1014                  *      1. jailed, non-wild.
1015                  *      2. jailed, wild.
1016                  *      3. non-jailed, non-wild.
1017                  *      4. non-jailed, wild.
1018                  */
1019                 head = &pcbgroup->ipg_hashbase[
1020                     INP_PCBHASH(INADDR_ANY, lport, 0, pcbgroup->ipg_hashmask)];
1021                 CK_LIST_FOREACH(inp, head, inp_pcbgrouphash) {
1022                         /* XXX inp locking */
1023                         if ((inp->inp_vflag & INP_IPV6) == 0)
1024                                 continue;
1025
1026                         if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) ||
1027                             inp->inp_lport != lport) {
1028                                 continue;
1029                         }
1030
1031                         injail = prison_flag(inp->inp_cred, PR_IP6);
1032                         if (injail) {
1033                                 if (prison_check_ip6(inp->inp_cred,
1034                                     laddr) != 0)
1035                                         continue;
1036                         } else {
1037                                 if (local_exact != NULL)
1038                                         continue;
1039                         }
1040
1041                         if (IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr)) {
1042                                 if (injail)
1043                                         goto found;
1044                                 else
1045                                         local_exact = inp;
1046                         } else if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
1047                                 if (injail)
1048                                         jail_wild = inp;
1049                                 else
1050                                         local_wild = inp;
1051                         }
1052                 } /* LIST_FOREACH */
1053
1054                 inp = jail_wild;
1055                 if (inp == NULL)
1056                         inp = jail_wild;
1057                 if (inp == NULL)
1058                         inp = local_exact;
1059                 if (inp == NULL)
1060                         inp = local_wild;
1061                 if (inp != NULL)
1062                         goto found;
1063         }
1064
1065         /*
1066          * Then look for a wildcard match, if requested.
1067          */
1068         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1069                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1070                 struct inpcb *jail_wild = NULL;
1071                 int injail;
1072
1073                 /*
1074                  * Order of socket selection - we always prefer jails.
1075                  *      1. jailed, non-wild.
1076                  *      2. jailed, wild.
1077                  *      3. non-jailed, non-wild.
1078                  *      4. non-jailed, wild.
1079                  */
1080                 head = &pcbinfo->ipi_wildbase[INP_PCBHASH(
1081                     INP6_PCBHASHKEY(&in6addr_any), lport, 0,
1082                     pcbinfo->ipi_wildmask)];
1083                 CK_LIST_FOREACH(inp, head, inp_pcbgroup_wild) {
1084                         /* XXX inp locking */
1085                         if ((inp->inp_vflag & INP_IPV6) == 0)
1086                                 continue;
1087
1088                         if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) ||
1089                             inp->inp_lport != lport) {
1090                                 continue;
1091                         }
1092
1093                         injail = prison_flag(inp->inp_cred, PR_IP6);
1094                         if (injail) {
1095                                 if (prison_check_ip6(inp->inp_cred,
1096                                     laddr) != 0)
1097                                         continue;
1098                         } else {
1099                                 if (local_exact != NULL)
1100                                         continue;
1101                         }
1102
1103                         if (IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr)) {
1104                                 if (injail)
1105                                         goto found;
1106                                 else
1107                                         local_exact = inp;
1108                         } else if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
1109                                 if (injail)
1110                                         jail_wild = inp;
1111                                 else
1112                                         local_wild = inp;
1113                         }
1114                 } /* LIST_FOREACH */
1115
1116                 inp = jail_wild;
1117                 if (inp == NULL)
1118                         inp = jail_wild;
1119                 if (inp == NULL)
1120                         inp = local_exact;
1121                 if (inp == NULL)
1122                         inp = local_wild;
1123                 if (inp != NULL)
1124                         goto found;
1125         } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
1126         INP_GROUP_UNLOCK(pcbgroup);
1127         return (NULL);
1128
1129 found:
1130         if (lookupflags & INPLOOKUP_WLOCKPCB)
1131                 locked = INP_TRY_WLOCK(inp);
1132         else if (lookupflags & INPLOOKUP_RLOCKPCB)
1133                 locked = INP_TRY_RLOCK(inp);
1134         else
1135                 panic("%s: locking buf", __func__);
1136         if (!locked)
1137                 in_pcbref(inp);
1138         INP_GROUP_UNLOCK(pcbgroup);
1139         if (!locked) {
1140                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
1141                         INP_WLOCK(inp);
1142                         if (in_pcbrele_wlocked(inp))
1143                                 return (NULL);
1144                 } else {
1145                         INP_RLOCK(inp);
1146                         if (in_pcbrele_rlocked(inp))
1147                                 return (NULL);
1148                 }
1149         }
1150 #ifdef INVARIANTS
1151         if (lookupflags & INPLOOKUP_WLOCKPCB)
1152                 INP_WLOCK_ASSERT(inp);
1153         else
1154                 INP_RLOCK_ASSERT(inp);
1155 #endif
1156         return (inp);
1157 }
1158 #endif /* PCBGROUP */
1159
1160 /*
1161  * Lookup PCB in hash list.  Used in in_pcb.c as well as here.
1162  */
1163 struct inpcb *
1164 in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
1165     u_int fport_arg, struct in6_addr *laddr, u_int lport_arg,
1166     int lookupflags, struct ifnet *ifp, uint8_t numa_domain)
1167 {
1168         struct inpcbhead *head;
1169         struct inpcb *inp, *tmpinp;
1170         u_short fport = fport_arg, lport = lport_arg;
1171
1172         KASSERT((lookupflags & ~(INPLOOKUP_WILDCARD)) == 0,
1173             ("%s: invalid lookup flags %d", __func__, lookupflags));
1174
1175         INP_HASH_LOCK_ASSERT(pcbinfo);
1176
1177         /*
1178          * First look for an exact match.
1179          */
1180         tmpinp = NULL;
1181         head = &pcbinfo->ipi_hashbase[INP_PCBHASH(
1182             INP6_PCBHASHKEY(faddr), lport, fport, pcbinfo->ipi_hashmask)];
1183         CK_LIST_FOREACH(inp, head, inp_hash) {
1184                 /* XXX inp locking */
1185                 if ((inp->inp_vflag & INP_IPV6) == 0)
1186                         continue;
1187                 if (IN6_ARE_ADDR_EQUAL(&inp->in6p_faddr, faddr) &&
1188                     IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr) &&
1189                     inp->inp_fport == fport &&
1190                     inp->inp_lport == lport) {
1191                         /*
1192                          * XXX We should be able to directly return
1193                          * the inp here, without any checks.
1194                          * Well unless both bound with SO_REUSEPORT?
1195                          */
1196                         if (prison_flag(inp->inp_cred, PR_IP6))
1197                                 return (inp);
1198                         if (tmpinp == NULL)
1199                                 tmpinp = inp;
1200                 }
1201         }
1202         if (tmpinp != NULL)
1203                 return (tmpinp);
1204
1205         /*
1206          * Then look in lb group (for wildcard match).
1207          */
1208         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1209                 inp = in6_pcblookup_lbgroup(pcbinfo, laddr, lport, faddr,
1210                     fport, lookupflags, numa_domain);
1211                 if (inp != NULL)
1212                         return (inp);
1213         }
1214
1215         /*
1216          * Then look for a wildcard match, if requested.
1217          */
1218         if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
1219                 struct inpcb *local_wild = NULL, *local_exact = NULL;
1220                 struct inpcb *jail_wild = NULL;
1221                 int injail;
1222
1223                 /*
1224                  * Order of socket selection - we always prefer jails.
1225                  *      1. jailed, non-wild.
1226                  *      2. jailed, wild.
1227                  *      3. non-jailed, non-wild.
1228                  *      4. non-jailed, wild.
1229                  */
1230                 head = &pcbinfo->ipi_hashbase[INP_PCBHASH(
1231                     INP6_PCBHASHKEY(&in6addr_any), lport, 0,
1232                     pcbinfo->ipi_hashmask)];
1233                 CK_LIST_FOREACH(inp, head, inp_hash) {
1234                         /* XXX inp locking */
1235                         if ((inp->inp_vflag & INP_IPV6) == 0)
1236                                 continue;
1237
1238                         if (!IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_faddr) ||
1239                             inp->inp_lport != lport) {
1240                                 continue;
1241                         }
1242
1243                         injail = prison_flag(inp->inp_cred, PR_IP6);
1244                         if (injail) {
1245                                 if (prison_check_ip6(inp->inp_cred,
1246                                     laddr) != 0)
1247                                         continue;
1248                         } else {
1249                                 if (local_exact != NULL)
1250                                         continue;
1251                         }
1252
1253                         if (IN6_ARE_ADDR_EQUAL(&inp->in6p_laddr, laddr)) {
1254                                 if (injail)
1255                                         return (inp);
1256                                 else
1257                                         local_exact = inp;
1258                         } else if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
1259                                 if (injail)
1260                                         jail_wild = inp;
1261                                 else
1262                                         local_wild = inp;
1263                         }
1264                 } /* LIST_FOREACH */
1265
1266                 if (jail_wild != NULL)
1267                         return (jail_wild);
1268                 if (local_exact != NULL)
1269                         return (local_exact);
1270                 if (local_wild != NULL)
1271                         return (local_wild);
1272         } /* if ((lookupflags & INPLOOKUP_WILDCARD) != 0) */
1273
1274         /*
1275          * Not found.
1276          */
1277         return (NULL);
1278 }
1279
1280 /*
1281  * Lookup PCB in hash list, using pcbinfo tables.  This variation locks the
1282  * hash list lock, and will return the inpcb locked (i.e., requires
1283  * INPLOOKUP_LOCKPCB).
1284  */
1285 static struct inpcb *
1286 in6_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
1287     u_int fport, struct in6_addr *laddr, u_int lport, int lookupflags,
1288     struct ifnet *ifp, uint8_t numa_domain)
1289 {
1290         struct inpcb *inp;
1291
1292         inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
1293             (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp,
1294             numa_domain);
1295         if (inp != NULL) {
1296                 if (lookupflags & INPLOOKUP_WLOCKPCB) {
1297                         INP_WLOCK(inp);
1298                         if (__predict_false(inp->inp_flags2 & INP_FREED)) {
1299                                 INP_WUNLOCK(inp);
1300                                 inp = NULL;
1301                         }
1302                 } else if (lookupflags & INPLOOKUP_RLOCKPCB) {
1303                         INP_RLOCK(inp);
1304                         if (__predict_false(inp->inp_flags2 & INP_FREED)) {
1305                                 INP_RUNLOCK(inp);
1306                                 inp = NULL;
1307                         }
1308                 } else
1309                         panic("%s: locking bug", __func__);
1310 #ifdef INVARIANTS
1311                 if (inp != NULL) {
1312                         if (lookupflags & INPLOOKUP_WLOCKPCB)
1313                                 INP_WLOCK_ASSERT(inp);
1314                         else
1315                                 INP_RLOCK_ASSERT(inp);
1316                 }
1317 #endif
1318         }
1319         return (inp);
1320 }
1321
1322 /*
1323  * Public inpcb lookup routines, accepting a 4-tuple, and optionally, an mbuf
1324  * from which a pre-calculated hash value may be extracted.
1325  *
1326  * Possibly more of this logic should be in in6_pcbgroup.c.
1327  */
1328 struct inpcb *
1329 in6_pcblookup(struct inpcbinfo *pcbinfo, struct in6_addr *faddr, u_int fport,
1330     struct in6_addr *laddr, u_int lport, int lookupflags, struct ifnet *ifp)
1331 {
1332 #if defined(PCBGROUP) && !defined(RSS)
1333         struct inpcbgroup *pcbgroup;
1334 #endif
1335
1336         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
1337             ("%s: invalid lookup flags %d", __func__, lookupflags));
1338         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
1339             ("%s: LOCKPCB not set", __func__));
1340
1341         /*
1342          * When not using RSS, use connection groups in preference to the
1343          * reservation table when looking up 4-tuples.  When using RSS, just
1344          * use the reservation table, due to the cost of the Toeplitz hash
1345          * in software.
1346          *
1347          * XXXRW: This policy belongs in the pcbgroup code, as in principle
1348          * we could be doing RSS with a non-Toeplitz hash that is affordable
1349          * in software.
1350          */
1351 #if defined(PCBGROUP) && !defined(RSS)
1352         if (in_pcbgroup_enabled(pcbinfo)) {
1353                 pcbgroup = in6_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
1354                     fport);
1355                 return (in6_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
1356                     laddr, lport, lookupflags, ifp));
1357         }
1358 #endif
1359         return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
1360             lookupflags, ifp, M_NODOM));
1361 }
1362
1363 struct inpcb *
1364 in6_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
1365     u_int fport, struct in6_addr *laddr, u_int lport, int lookupflags,
1366     struct ifnet *ifp, struct mbuf *m)
1367 {
1368 #ifdef PCBGROUP
1369         struct inpcbgroup *pcbgroup;
1370 #endif
1371
1372         KASSERT((lookupflags & ~INPLOOKUP_MASK) == 0,
1373             ("%s: invalid lookup flags %d", __func__, lookupflags));
1374         KASSERT((lookupflags & (INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)) != 0,
1375             ("%s: LOCKPCB not set", __func__));
1376
1377 #ifdef PCBGROUP
1378         /*
1379          * If we can use a hardware-generated hash to look up the connection
1380          * group, use that connection group to find the inpcb.  Otherwise
1381          * fall back on a software hash -- or the reservation table if we're
1382          * using RSS.
1383          *
1384          * XXXRW: As above, that policy belongs in the pcbgroup code.
1385          */
1386         if (in_pcbgroup_enabled(pcbinfo) &&
1387             M_HASHTYPE_TEST(m, M_HASHTYPE_NONE) == 0) {
1388                 pcbgroup = in6_pcbgroup_byhash(pcbinfo, M_HASHTYPE_GET(m),
1389                     m->m_pkthdr.flowid);
1390                 if (pcbgroup != NULL)
1391                         return (in6_pcblookup_group(pcbinfo, pcbgroup, faddr,
1392                             fport, laddr, lport, lookupflags, ifp));
1393 #ifndef RSS
1394                 pcbgroup = in6_pcbgroup_bytuple(pcbinfo, laddr, lport, faddr,
1395                     fport);
1396                 return (in6_pcblookup_group(pcbinfo, pcbgroup, faddr, fport,
1397                     laddr, lport, lookupflags, ifp));
1398 #endif
1399         }
1400 #endif
1401         return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
1402             lookupflags, ifp, m->m_pkthdr.numa_domain));
1403 }
1404
1405 void
1406 init_sin6(struct sockaddr_in6 *sin6, struct mbuf *m, int srcordst)
1407 {
1408         struct ip6_hdr *ip;
1409
1410         ip = mtod(m, struct ip6_hdr *);
1411         bzero(sin6, sizeof(*sin6));
1412         sin6->sin6_len = sizeof(*sin6);
1413         sin6->sin6_family = AF_INET6;
1414         sin6->sin6_addr = srcordst ? ip->ip6_dst : ip->ip6_src;
1415
1416         (void)sa6_recoverscope(sin6); /* XXX: should catch errors... */
1417
1418         return;
1419 }