sys/netinet/tcp_usrreq.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 1982, 1986, 1988, 1993
   5  *      The Regents of the University of California.
   6  * Copyright (c) 2006-2007 Robert N. M. Watson
   7  * Copyright (c) 2010-2011 Juniper Networks, Inc.
   8  * All rights reserved.
   9  *
  10  * Portions of this software were developed by Robert N. M. Watson under
  11  * contract to Juniper Networks, Inc.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  * 1. Redistributions of source code must retain the above copyright
  17  *    notice, this list of conditions and the following disclaimer.
  18  * 2. Redistributions in binary form must reproduce the above copyright
  19  *    notice, this list of conditions and the following disclaimer in the
  20  *    documentation and/or other materials provided with the distribution.
  21  * 3. Neither the name of the University nor the names of its contributors
  22  *    may be used to endorse or promote products derived from this software
  23  *    without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  35  * SUCH DAMAGE.
  36  *
  37  *      From: @(#)tcp_usrreq.c  8.2 (Berkeley) 1/3/94
  38  */
  39
  40 #include <sys/cdefs.h>
  41 __FBSDID("$FreeBSD$");
  42
  43 #include "opt_ddb.h"
  44 #include "opt_inet.h"
  45 #include "opt_inet6.h"
  46 #include "opt_ipsec.h"
  47 #include "opt_kern_tls.h"
  48
  49 #include <sys/param.h>
  50 #include <sys/systm.h>
  51 #include <sys/arb.h>
  52 #include <sys/limits.h>
  53 #include <sys/malloc.h>
  54 #include <sys/refcount.h>
  55 #include <sys/kernel.h>
  56 #include <sys/ktls.h>
  57 #include <sys/qmath.h>
  58 #include <sys/sysctl.h>
  59 #include <sys/mbuf.h>
  60 #ifdef INET6
  61 #include <sys/domain.h>
  62 #endif /* INET6 */
  63 #include <sys/socket.h>
  64 #include <sys/socketvar.h>
  65 #include <sys/protosw.h>
  66 #include <sys/proc.h>
  67 #include <sys/jail.h>
  68 #include <sys/stats.h>
  69
  70 #ifdef DDB
  71 #include <ddb/ddb.h>
  72 #endif
  73
  74 #include <net/if.h>
  75 #include <net/if_var.h>
  76 #include <net/route.h>
  77 #include <net/vnet.h>
  78
  79 #include <netinet/in.h>
  80 #include <netinet/in_kdtrace.h>
  81 #include <netinet/in_pcb.h>
  82 #include <netinet/in_systm.h>
  83 #include <netinet/in_var.h>
  84 #include <netinet/ip.h>
  85 #include <netinet/ip_var.h>
  86 #ifdef INET6
  87 #include <netinet/ip6.h>
  88 #include <netinet6/in6_pcb.h>
  89 #include <netinet6/ip6_var.h>
  90 #include <netinet6/scope6_var.h>
  91 #endif
  92 #include <netinet/tcp.h>
  93 #include <netinet/tcp_fsm.h>
  94 #include <netinet/tcp_seq.h>
  95 #include <netinet/tcp_timer.h>
  96 #include <netinet/tcp_var.h>
  97 #include <netinet/tcp_log_buf.h>
  98 #include <netinet/tcpip.h>
  99 #include <netinet/cc/cc.h>
 100 #include <netinet/tcp_fastopen.h>
 101 #include <netinet/tcp_hpts.h>
 102 #ifdef TCPPCAP
 103 #include <netinet/tcp_pcap.h>
 104 #endif
 105 #ifdef TCP_OFFLOAD
 106 #include <netinet/tcp_offload.h>
 107 #endif
 108 #include <netipsec/ipsec_support.h>
 109
 110 #include <vm/vm.h>
 111 #include <vm/vm_param.h>
 112 #include <vm/pmap.h>
 113 #include <vm/vm_extern.h>
 114 #include <vm/vm_map.h>
 115 #include <vm/vm_page.h>
 116
 117 /*
 118  * TCP protocol interface to socket abstraction.
 119  */
 120 #ifdef INET
 121 static int      tcp_connect(struct tcpcb *, struct sockaddr *,
 122                     struct thread *td);
 123 #endif /* INET */
 124 #ifdef INET6
 125 static int      tcp6_connect(struct tcpcb *, struct sockaddr *,
 126                     struct thread *td);
 127 #endif /* INET6 */
 128 static void     tcp_disconnect(struct tcpcb *);
 129 static void     tcp_usrclosed(struct tcpcb *);
 130 static void     tcp_fill_info(struct tcpcb *, struct tcp_info *);
 131
 132 static int      tcp_pru_options_support(struct tcpcb *tp, int flags);
 133
 134 /*
 135  * TCP attaches to socket via pru_attach(), reserving space,
 136  * and an internet control block.
 137  */
 138 static int
 139 tcp_usr_attach(struct socket *so, int proto, struct thread *td)
 140 {
 141         struct inpcb *inp;
 142         struct tcpcb *tp = NULL;
 143         int error;
 144
 145         inp = sotoinpcb(so);
 146         KASSERT(inp == NULL, ("tcp_usr_attach: inp != NULL"));
 147
 148         error = soreserve(so, V_tcp_sendspace, V_tcp_recvspace);
 149         if (error)
 150                 goto out;
 151
 152         so->so_rcv.sb_flags |= SB_AUTOSIZE;
 153         so->so_snd.sb_flags |= SB_AUTOSIZE;
 154         error = in_pcballoc(so, &V_tcbinfo);
 155         if (error)
 156                 goto out;
 157         inp = sotoinpcb(so);
 158         tp = tcp_newtcpcb(inp);
 159         if (tp == NULL) {
 160                 error = ENOBUFS;
 161                 in_pcbdetach(inp);
 162                 in_pcbfree(inp);
 163                 goto out;
 164         }
 165         tp->t_state = TCPS_CLOSED;
 166         INP_WUNLOCK(inp);
 167         TCPSTATES_INC(TCPS_CLOSED);
 168 out:
 169         TCP_PROBE2(debug__user, tp, PRU_ATTACH);
 170         return (error);
 171 }
 172
 173 /*
 174  * tcp_usr_detach is called when the socket layer loses its final reference
 175  * to the socket, be it a file descriptor reference, a reference from TCP,
 176  * etc.  At this point, there is only one case in which we will keep around
 177  * inpcb state: time wait.
 178  */
 179 static void
 180 tcp_usr_detach(struct socket *so)
 181 {
 182         struct inpcb *inp;
 183         struct tcpcb *tp;
 184
 185         inp = sotoinpcb(so);
 186         KASSERT(inp != NULL, ("%s: inp == NULL", __func__));
 187         INP_WLOCK(inp);
 188         KASSERT(so->so_pcb == inp && inp->inp_socket == so,
 189                 ("%s: socket %p inp %p mismatch", __func__, so, inp));
 190
 191         tp = intotcpcb(inp);
 192
 193         KASSERT(inp->inp_flags & INP_DROPPED ||
 194             tp->t_state < TCPS_SYN_SENT,
 195             ("%s: inp %p not dropped or embryonic", __func__, inp));
 196
 197         tcp_discardcb(tp);
 198         in_pcbdetach(inp);
 199         in_pcbfree(inp);
 200 }
 201
 202 #ifdef INET
 203 /*
 204  * Give the socket an address.
 205  */
 206 static int
 207 tcp_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 208 {
 209         int error = 0;
 210         struct inpcb *inp;
 211 #ifdef KDTRACE_HOOKS
 212         struct tcpcb *tp = NULL;
 213 #endif
 214         struct sockaddr_in *sinp;
 215
 216         sinp = (struct sockaddr_in *)nam;
 217         if (nam->sa_family != AF_INET) {
 218                 /*
 219                  * Preserve compatibility with old programs.
 220                  */
 221                 if (nam->sa_family != AF_UNSPEC ||
 222                     nam->sa_len < offsetof(struct sockaddr_in, sin_zero) ||
 223                     sinp->sin_addr.s_addr != INADDR_ANY)
 224                         return (EAFNOSUPPORT);
 225                 nam->sa_family = AF_INET;
 226         }
 227         if (nam->sa_len != sizeof(*sinp))
 228                 return (EINVAL);
 229
 230         /*
 231          * Must check for multicast addresses and disallow binding
 232          * to them.
 233          */
 234         if (IN_MULTICAST(ntohl(sinp->sin_addr.s_addr)))
 235                 return (EAFNOSUPPORT);
 236
 237         inp = sotoinpcb(so);
 238         KASSERT(inp != NULL, ("tcp_usr_bind: inp == NULL"));
 239         INP_WLOCK(inp);
 240         if (inp->inp_flags & INP_DROPPED) {
 241                 error = EINVAL;
 242                 goto out;
 243         }
 244 #ifdef KDTRACE_HOOKS
 245         tp = intotcpcb(inp);
 246 #endif
 247         INP_HASH_WLOCK(&V_tcbinfo);
 248         error = in_pcbbind(inp, nam, td->td_ucred);
 249         INP_HASH_WUNLOCK(&V_tcbinfo);
 250 out:
 251         TCP_PROBE2(debug__user, tp, PRU_BIND);
 252         INP_WUNLOCK(inp);
 253
 254         return (error);
 255 }
 256 #endif /* INET */
 257
 258 #ifdef INET6
 259 static int
 260 tcp6_usr_bind(struct socket *so, struct sockaddr *nam, struct thread *td)
 261 {
 262         int error = 0;
 263         struct inpcb *inp;
 264 #ifdef KDTRACE_HOOKS
 265         struct tcpcb *tp = NULL;
 266 #endif
 267         struct sockaddr_in6 *sin6;
 268         u_char vflagsav;
 269
 270         sin6 = (struct sockaddr_in6 *)nam;
 271         if (nam->sa_family != AF_INET6)
 272                 return (EAFNOSUPPORT);
 273         if (nam->sa_len != sizeof(*sin6))
 274                 return (EINVAL);
 275
 276         /*
 277          * Must check for multicast addresses and disallow binding
 278          * to them.
 279          */
 280         if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr))
 281                 return (EAFNOSUPPORT);
 282
 283         inp = sotoinpcb(so);
 284         KASSERT(inp != NULL, ("tcp6_usr_bind: inp == NULL"));
 285         INP_WLOCK(inp);
 286         vflagsav = inp->inp_vflag;
 287         if (inp->inp_flags & INP_DROPPED) {
 288                 error = EINVAL;
 289                 goto out;
 290         }
 291 #ifdef KDTRACE_HOOKS
 292         tp = intotcpcb(inp);
 293 #endif
 294         INP_HASH_WLOCK(&V_tcbinfo);
 295         inp->inp_vflag &= ~INP_IPV4;
 296         inp->inp_vflag |= INP_IPV6;
 297 #ifdef INET
 298         if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0) {
 299                 if (IN6_IS_ADDR_UNSPECIFIED(&sin6->sin6_addr))
 300                         inp->inp_vflag |= INP_IPV4;
 301                 else if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
 302                         struct sockaddr_in sin;
 303
 304                         in6_sin6_2_sin(&sin, sin6);
 305                         if (IN_MULTICAST(ntohl(sin.sin_addr.s_addr))) {
 306                                 error = EAFNOSUPPORT;
 307                                 INP_HASH_WUNLOCK(&V_tcbinfo);
 308                                 goto out;
 309                         }
 310                         inp->inp_vflag |= INP_IPV4;
 311                         inp->inp_vflag &= ~INP_IPV6;
 312                         error = in_pcbbind(inp, (struct sockaddr *)&sin,
 313                             td->td_ucred);
 314                         INP_HASH_WUNLOCK(&V_tcbinfo);
 315                         goto out;
 316                 }
 317         }
 318 #endif
 319         error = in6_pcbbind(inp, nam, td->td_ucred);
 320         INP_HASH_WUNLOCK(&V_tcbinfo);
 321 out:
 322         if (error != 0)
 323                 inp->inp_vflag = vflagsav;
 324         TCP_PROBE2(debug__user, tp, PRU_BIND);
 325         INP_WUNLOCK(inp);
 326         return (error);
 327 }
 328 #endif /* INET6 */
 329
 330 #ifdef INET
 331 /*
 332  * Prepare to accept connections.
 333  */
 334 static int
 335 tcp_usr_listen(struct socket *so, int backlog, struct thread *td)
 336 {
 337         int error = 0;
 338         struct inpcb *inp;
 339         struct tcpcb *tp = NULL;
 340
 341         inp = sotoinpcb(so);
 342         KASSERT(inp != NULL, ("tcp_usr_listen: inp == NULL"));
 343         INP_WLOCK(inp);
 344         if (inp->inp_flags & INP_DROPPED) {
 345                 error = EINVAL;
 346                 goto out;
 347         }
 348         tp = intotcpcb(inp);
 349         SOCK_LOCK(so);
 350         error = solisten_proto_check(so);
 351         if (error != 0) {
 352                 SOCK_UNLOCK(so);
 353                 goto out;
 354         }
 355         if (inp->inp_lport == 0) {
 356                 INP_HASH_WLOCK(&V_tcbinfo);
 357                 error = in_pcbbind(inp, NULL, td->td_ucred);
 358                 INP_HASH_WUNLOCK(&V_tcbinfo);
 359         }
 360         if (error == 0) {
 361                 tcp_state_change(tp, TCPS_LISTEN);
 362                 solisten_proto(so, backlog);
 363 #ifdef TCP_OFFLOAD
 364                 if ((so->so_options & SO_NO_OFFLOAD) == 0)
 365                         tcp_offload_listen_start(tp);
 366 #endif
 367         } else {
 368                 solisten_proto_abort(so);
 369         }
 370         SOCK_UNLOCK(so);
 371
 372         if (IS_FASTOPEN(tp->t_flags))
 373                 tp->t_tfo_pending = tcp_fastopen_alloc_counter();
 374
 375 out:
 376         TCP_PROBE2(debug__user, tp, PRU_LISTEN);
 377         INP_WUNLOCK(inp);
 378         return (error);
 379 }
 380 #endif /* INET */
 381
 382 #ifdef INET6
 383 static int
 384 tcp6_usr_listen(struct socket *so, int backlog, struct thread *td)
 385 {
 386         int error = 0;
 387         struct inpcb *inp;
 388         struct tcpcb *tp = NULL;
 389         u_char vflagsav;
 390
 391         inp = sotoinpcb(so);
 392         KASSERT(inp != NULL, ("tcp6_usr_listen: inp == NULL"));
 393         INP_WLOCK(inp);
 394         if (inp->inp_flags & INP_DROPPED) {
 395                 error = EINVAL;
 396                 goto out;
 397         }
 398         vflagsav = inp->inp_vflag;
 399         tp = intotcpcb(inp);
 400         SOCK_LOCK(so);
 401         error = solisten_proto_check(so);
 402         if (error != 0) {
 403                 SOCK_UNLOCK(so);
 404                 goto out;
 405         }
 406         INP_HASH_WLOCK(&V_tcbinfo);
 407         if (inp->inp_lport == 0) {
 408                 inp->inp_vflag &= ~INP_IPV4;
 409                 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0)
 410                         inp->inp_vflag |= INP_IPV4;
 411                 error = in6_pcbbind(inp, NULL, td->td_ucred);
 412         }
 413         INP_HASH_WUNLOCK(&V_tcbinfo);
 414         if (error == 0) {
 415                 tcp_state_change(tp, TCPS_LISTEN);
 416                 solisten_proto(so, backlog);
 417 #ifdef TCP_OFFLOAD
 418                 if ((so->so_options & SO_NO_OFFLOAD) == 0)
 419                         tcp_offload_listen_start(tp);
 420 #endif
 421         } else {
 422                 solisten_proto_abort(so);
 423         }
 424         SOCK_UNLOCK(so);
 425
 426         if (IS_FASTOPEN(tp->t_flags))
 427                 tp->t_tfo_pending = tcp_fastopen_alloc_counter();
 428
 429         if (error != 0)
 430                 inp->inp_vflag = vflagsav;
 431
 432 out:
 433         TCP_PROBE2(debug__user, tp, PRU_LISTEN);
 434         INP_WUNLOCK(inp);
 435         return (error);
 436 }
 437 #endif /* INET6 */
 438
 439 #ifdef INET
 440 /*
 441  * Initiate connection to peer.
 442  * Create a template for use in transmissions on this connection.
 443  * Enter SYN_SENT state, and mark socket as connecting.
 444  * Start keep-alive timer, and seed output sequence space.
 445  * Send initial segment on connection.
 446  */
 447 static int
 448 tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 449 {
 450         struct epoch_tracker et;
 451         int error = 0;
 452         struct inpcb *inp;
 453         struct tcpcb *tp = NULL;
 454         struct sockaddr_in *sinp;
 455
 456         sinp = (struct sockaddr_in *)nam;
 457         if (nam->sa_family != AF_INET)
 458                 return (EAFNOSUPPORT);
 459         if (nam->sa_len != sizeof (*sinp))
 460                 return (EINVAL);
 461
 462         /*
 463          * Must disallow TCP ``connections'' to multicast addresses.
 464          */
 465         if (IN_MULTICAST(ntohl(sinp->sin_addr.s_addr)))
 466                 return (EAFNOSUPPORT);
 467         if (ntohl(sinp->sin_addr.s_addr) == INADDR_BROADCAST)
 468                 return (EACCES);
 469         if ((error = prison_remote_ip4(td->td_ucred, &sinp->sin_addr)) != 0)
 470                 return (error);
 471
 472         inp = sotoinpcb(so);
 473         KASSERT(inp != NULL, ("tcp_usr_connect: inp == NULL"));
 474         INP_WLOCK(inp);
 475         if (inp->inp_flags & INP_DROPPED) {
 476                 error = ECONNREFUSED;
 477                 goto out;
 478         }
 479         if (SOLISTENING(so)) {
 480                 error = EOPNOTSUPP;
 481                 goto out;
 482         }
 483         tp = intotcpcb(inp);
 484         NET_EPOCH_ENTER(et);
 485         if ((error = tcp_connect(tp, nam, td)) != 0)
 486                 goto out_in_epoch;
 487 #ifdef TCP_OFFLOAD
 488         if (registered_toedevs > 0 &&
 489             (so->so_options & SO_NO_OFFLOAD) == 0 &&
 490             (error = tcp_offload_connect(so, nam)) == 0)
 491                 goto out_in_epoch;
 492 #endif
 493         tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
 494         error = tcp_output(tp);
 495         KASSERT(error >= 0, ("TCP stack %s requested tcp_drop(%p) at connect()"
 496             ", error code %d", tp->t_fb->tfb_tcp_block_name, tp, -error));
 497 out_in_epoch:
 498         NET_EPOCH_EXIT(et);
 499 out:
 500         TCP_PROBE2(debug__user, tp, PRU_CONNECT);
 501         INP_WUNLOCK(inp);
 502         return (error);
 503 }
 504 #endif /* INET */
 505
 506 #ifdef INET6
 507 static int
 508 tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
 509 {
 510         struct epoch_tracker et;
 511         int error = 0;
 512         struct inpcb *inp;
 513         struct tcpcb *tp = NULL;
 514         struct sockaddr_in6 *sin6;
 515         u_int8_t incflagsav;
 516         u_char vflagsav;
 517
 518         sin6 = (struct sockaddr_in6 *)nam;
 519         if (nam->sa_family != AF_INET6)
 520                 return (EAFNOSUPPORT);
 521         if (nam->sa_len != sizeof (*sin6))
 522                 return (EINVAL);
 523
 524         /*
 525          * Must disallow TCP ``connections'' to multicast addresses.
 526          */
 527         if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr))
 528                 return (EAFNOSUPPORT);
 529
 530         inp = sotoinpcb(so);
 531         KASSERT(inp != NULL, ("tcp6_usr_connect: inp == NULL"));
 532         INP_WLOCK(inp);
 533         vflagsav = inp->inp_vflag;
 534         incflagsav = inp->inp_inc.inc_flags;
 535         if (inp->inp_flags & INP_DROPPED) {
 536                 error = ECONNREFUSED;
 537                 goto out;
 538         }
 539         if (SOLISTENING(so)) {
 540                 error = EINVAL;
 541                 goto out;
 542         }
 543         tp = intotcpcb(inp);
 544 #ifdef INET
 545         /*
 546          * XXXRW: Some confusion: V4/V6 flags relate to binding, and
 547          * therefore probably require the hash lock, which isn't held here.
 548          * Is this a significant problem?
 549          */
 550         if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
 551                 struct sockaddr_in sin;
 552
 553                 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0) {
 554                         error = EINVAL;
 555                         goto out;
 556                 }
 557                 if ((inp->inp_vflag & INP_IPV4) == 0) {
 558                         error = EAFNOSUPPORT;
 559                         goto out;
 560                 }
 561
 562                 in6_sin6_2_sin(&sin, sin6);
 563                 if (IN_MULTICAST(ntohl(sin.sin_addr.s_addr))) {
 564                         error = EAFNOSUPPORT;
 565                         goto out;
 566                 }
 567                 if (ntohl(sin.sin_addr.s_addr) == INADDR_BROADCAST) {
 568                         error = EACCES;
 569                         goto out;
 570                 }
 571                 if ((error = prison_remote_ip4(td->td_ucred,
 572                     &sin.sin_addr)) != 0)
 573                         goto out;
 574                 inp->inp_vflag |= INP_IPV4;
 575                 inp->inp_vflag &= ~INP_IPV6;
 576                 NET_EPOCH_ENTER(et);
 577                 if ((error = tcp_connect(tp, (struct sockaddr *)&sin, td)) != 0)
 578                         goto out_in_epoch;
 579 #ifdef TCP_OFFLOAD
 580                 if (registered_toedevs > 0 &&
 581                     (so->so_options & SO_NO_OFFLOAD) == 0 &&
 582                     (error = tcp_offload_connect(so, nam)) == 0)
 583                         goto out_in_epoch;
 584 #endif
 585                 error = tcp_output(tp);
 586                 goto out_in_epoch;
 587         } else {
 588                 if ((inp->inp_vflag & INP_IPV6) == 0) {
 589                         error = EAFNOSUPPORT;
 590                         goto out;
 591                 }
 592         }
 593 #endif
 594         if ((error = prison_remote_ip6(td->td_ucred, &sin6->sin6_addr)) != 0)
 595                 goto out;
 596         inp->inp_vflag &= ~INP_IPV4;
 597         inp->inp_vflag |= INP_IPV6;
 598         inp->inp_inc.inc_flags |= INC_ISIPV6;
 599         NET_EPOCH_ENTER(et);
 600         if ((error = tcp6_connect(tp, nam, td)) != 0)
 601                 goto out_in_epoch;
 602 #ifdef TCP_OFFLOAD
 603         if (registered_toedevs > 0 &&
 604             (so->so_options & SO_NO_OFFLOAD) == 0 &&
 605             (error = tcp_offload_connect(so, nam)) == 0)
 606                 goto out_in_epoch;
 607 #endif
 608         tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp));
 609         error = tcp_output(tp);
 610 out_in_epoch:
 611         NET_EPOCH_EXIT(et);
 612 out:
 613         KASSERT(error >= 0, ("TCP stack %s requested tcp_drop(%p) at connect()"
 614             ", error code %d", tp->t_fb->tfb_tcp_block_name, tp, -error));
 615         /*
 616          * If the implicit bind in the connect call fails, restore
 617          * the flags we modified.
 618          */
 619         if (error != 0 && inp->inp_lport == 0) {
 620                 inp->inp_vflag = vflagsav;
 621                 inp->inp_inc.inc_flags = incflagsav;
 622         }
 623
 624         TCP_PROBE2(debug__user, tp, PRU_CONNECT);
 625         INP_WUNLOCK(inp);
 626         return (error);
 627 }
 628 #endif /* INET6 */
 629
 630 /*
 631  * Initiate disconnect from peer.
 632  * If connection never passed embryonic stage, just drop;
 633  * else if don't need to let data drain, then can just drop anyways,
 634  * else have to begin TCP shutdown process: mark socket disconnecting,
 635  * drain unread data, state switch to reflect user close, and
 636  * send segment (e.g. FIN) to peer.  Socket will be really disconnected
 637  * when peer sends FIN and acks ours.
 638  *
 639  * SHOULD IMPLEMENT LATER PRU_CONNECT VIA REALLOC TCPCB.
 640  */
 641 static int
 642 tcp_usr_disconnect(struct socket *so)
 643 {
 644         struct inpcb *inp;
 645         struct tcpcb *tp = NULL;
 646         struct epoch_tracker et;
 647         int error = 0;
 648
 649         NET_EPOCH_ENTER(et);
 650         inp = sotoinpcb(so);
 651         KASSERT(inp != NULL, ("tcp_usr_disconnect: inp == NULL"));
 652         INP_WLOCK(inp);
 653         if (inp->inp_flags & INP_DROPPED) {
 654                 error = ECONNRESET;
 655                 goto out;
 656         }
 657         tp = intotcpcb(inp);
 658         tcp_disconnect(tp);
 659 out:
 660         TCP_PROBE2(debug__user, tp, PRU_DISCONNECT);
 661         INP_WUNLOCK(inp);
 662         NET_EPOCH_EXIT(et);
 663         return (error);
 664 }
 665
 666 #ifdef INET
 667 /*
 668  * Accept a connection.  Essentially all the work is done at higher levels;
 669  * just return the address of the peer, storing through addr.
 670  */
 671 static int
 672 tcp_usr_accept(struct socket *so, struct sockaddr **nam)
 673 {
 674         int error = 0;
 675         struct inpcb *inp = NULL;
 676 #ifdef KDTRACE_HOOKS
 677         struct tcpcb *tp = NULL;
 678 #endif
 679         struct in_addr addr;
 680         in_port_t port = 0;
 681
 682         if (so->so_state & SS_ISDISCONNECTED)
 683                 return (ECONNABORTED);
 684
 685         inp = sotoinpcb(so);
 686         KASSERT(inp != NULL, ("tcp_usr_accept: inp == NULL"));
 687         INP_WLOCK(inp);
 688         if (inp->inp_flags & INP_DROPPED) {
 689                 error = ECONNABORTED;
 690                 goto out;
 691         }
 692 #ifdef KDTRACE_HOOKS
 693         tp = intotcpcb(inp);
 694 #endif
 695
 696         /*
 697          * We inline in_getpeeraddr and COMMON_END here, so that we can
 698          * copy the data of interest and defer the malloc until after we
 699          * release the lock.
 700          */
 701         port = inp->inp_fport;
 702         addr = inp->inp_faddr;
 703
 704 out:
 705         TCP_PROBE2(debug__user, tp, PRU_ACCEPT);
 706         INP_WUNLOCK(inp);
 707         if (error == 0)
 708                 *nam = in_sockaddr(port, &addr);
 709         return error;
 710 }
 711 #endif /* INET */
 712
 713 #ifdef INET6
 714 static int
 715 tcp6_usr_accept(struct socket *so, struct sockaddr **nam)
 716 {
 717         struct inpcb *inp = NULL;
 718         int error = 0;
 719 #ifdef KDTRACE_HOOKS
 720         struct tcpcb *tp = NULL;
 721 #endif
 722         struct in_addr addr;
 723         struct in6_addr addr6;
 724         struct epoch_tracker et;
 725         in_port_t port = 0;
 726         int v4 = 0;
 727
 728         if (so->so_state & SS_ISDISCONNECTED)
 729                 return (ECONNABORTED);
 730
 731         inp = sotoinpcb(so);
 732         KASSERT(inp != NULL, ("tcp6_usr_accept: inp == NULL"));
 733         NET_EPOCH_ENTER(et);
 734         INP_WLOCK(inp);
 735         if (inp->inp_flags & INP_DROPPED) {
 736                 error = ECONNABORTED;
 737                 goto out;
 738         }
 739 #ifdef KDTRACE_HOOKS
 740         tp = intotcpcb(inp);
 741 #endif
 742
 743         /*
 744          * We inline in6_mapped_peeraddr and COMMON_END here, so that we can
 745          * copy the data of interest and defer the malloc until after we
 746          * release the lock.
 747          */
 748         if (inp->inp_vflag & INP_IPV4) {
 749                 v4 = 1;
 750                 port = inp->inp_fport;
 751                 addr = inp->inp_faddr;
 752         } else {
 753                 port = inp->inp_fport;
 754                 addr6 = inp->in6p_faddr;
 755         }
 756
 757 out:
 758         TCP_PROBE2(debug__user, tp, PRU_ACCEPT);
 759         INP_WUNLOCK(inp);
 760         NET_EPOCH_EXIT(et);
 761         if (error == 0) {
 762                 if (v4)
 763                         *nam = in6_v4mapsin6_sockaddr(port, &addr);
 764                 else
 765                         *nam = in6_sockaddr(port, &addr6);
 766         }
 767         return error;
 768 }
 769 #endif /* INET6 */
 770
 771 /*
 772  * Mark the connection as being incapable of further output.
 773  */
 774 static int
 775 tcp_usr_shutdown(struct socket *so)
 776 {
 777         int error = 0;
 778         struct inpcb *inp;
 779         struct tcpcb *tp = NULL;
 780         struct epoch_tracker et;
 781
 782         inp = sotoinpcb(so);
 783         KASSERT(inp != NULL, ("inp == NULL"));
 784         INP_WLOCK(inp);
 785         if (inp->inp_flags & INP_DROPPED) {
 786                 INP_WUNLOCK(inp);
 787                 return (ECONNRESET);
 788         }
 789         tp = intotcpcb(inp);
 790         NET_EPOCH_ENTER(et);
 791         socantsendmore(so);
 792         tcp_usrclosed(tp);
 793         if (!(inp->inp_flags & INP_DROPPED))
 794                 error = tcp_output_nodrop(tp);
 795         TCP_PROBE2(debug__user, tp, PRU_SHUTDOWN);
 796         error = tcp_unlock_or_drop(tp, error);
 797         NET_EPOCH_EXIT(et);
 798
 799         return (error);
 800 }
 801
 802 /*
 803  * After a receive, possibly send window update to peer.
 804  */
 805 static int
 806 tcp_usr_rcvd(struct socket *so, int flags)
 807 {
 808         struct epoch_tracker et;
 809         struct inpcb *inp;
 810         struct tcpcb *tp = NULL;
 811         int outrv = 0, error = 0;
 812
 813         inp = sotoinpcb(so);
 814         KASSERT(inp != NULL, ("tcp_usr_rcvd: inp == NULL"));
 815         INP_WLOCK(inp);
 816         if (inp->inp_flags & INP_DROPPED) {
 817                 INP_WUNLOCK(inp);
 818                 return (ECONNRESET);
 819         }
 820         tp = intotcpcb(inp);
 821         NET_EPOCH_ENTER(et);
 822         /*
 823          * For passively-created TFO connections, don't attempt a window
 824          * update while still in SYN_RECEIVED as this may trigger an early
 825          * SYN|ACK.  It is preferable to have the SYN|ACK be sent along with
 826          * application response data, or failing that, when the DELACK timer
 827          * expires.
 828          */
 829         if (IS_FASTOPEN(tp->t_flags) &&
 830             (tp->t_state == TCPS_SYN_RECEIVED))
 831                 goto out;
 832 #ifdef TCP_OFFLOAD
 833         if (tp->t_flags & TF_TOE)
 834                 tcp_offload_rcvd(tp);
 835         else
 836 #endif
 837                 outrv = tcp_output_nodrop(tp);
 838 out:
 839         TCP_PROBE2(debug__user, tp, PRU_RCVD);
 840         (void) tcp_unlock_or_drop(tp, outrv);
 841         NET_EPOCH_EXIT(et);
 842         return (error);
 843 }
 844
 845 /*
 846  * Do a send by putting data in output queue and updating urgent
 847  * marker if URG set.  Possibly send more data.  Unlike the other
 848  * pru_*() routines, the mbuf chains are our responsibility.  We
 849  * must either enqueue them or free them.  The other pru_* routines
 850  * generally are caller-frees.
 851  */
 852 static int
 853 tcp_usr_send(struct socket *so, int flags, struct mbuf *m,
 854     struct sockaddr *nam, struct mbuf *control, struct thread *td)
 855 {
 856         struct epoch_tracker et;
 857         int error = 0;
 858         struct inpcb *inp;
 859         struct tcpcb *tp = NULL;
 860 #ifdef INET
 861 #ifdef INET6
 862         struct sockaddr_in sin;
 863 #endif
 864         struct sockaddr_in *sinp;
 865 #endif
 866 #ifdef INET6
 867         int isipv6;
 868 #endif
 869         u_int8_t incflagsav;
 870         u_char vflagsav;
 871         bool restoreflags;
 872
 873         if (control != NULL) {
 874                 /* TCP doesn't do control messages (rights, creds, etc) */
 875                 if (control->m_len) {
 876                         m_freem(control);
 877                         return (EINVAL);
 878                 }
 879                 m_freem(control);       /* empty control, just free it */
 880         }
 881
 882         inp = sotoinpcb(so);
 883         KASSERT(inp != NULL, ("tcp_usr_send: inp == NULL"));
 884         INP_WLOCK(inp);
 885         if (inp->inp_flags & INP_DROPPED) {
 886                 if (m != NULL && (flags & PRUS_NOTREADY) == 0)
 887                         m_freem(m);
 888                 INP_WUNLOCK(inp);
 889                 return (ECONNRESET);
 890         }
 891
 892         vflagsav = inp->inp_vflag;
 893         incflagsav = inp->inp_inc.inc_flags;
 894         restoreflags = false;
 895         tp = intotcpcb(inp);
 896
 897         NET_EPOCH_ENTER(et);
 898         if ((flags & PRUS_OOB) != 0 &&
 899             (error = tcp_pru_options_support(tp, PRUS_OOB)) != 0)
 900                 goto out;
 901
 902         if (nam != NULL && tp->t_state < TCPS_SYN_SENT) {
 903                 if (tp->t_state == TCPS_LISTEN) {
 904                         error = EINVAL;
 905                         goto out;
 906                 }
 907                 switch (nam->sa_family) {
 908 #ifdef INET
 909                 case AF_INET:
 910                         sinp = (struct sockaddr_in *)nam;
 911                         if (sinp->sin_len != sizeof(struct sockaddr_in)) {
 912                                 error = EINVAL;
 913                                 goto out;
 914                         }
 915                         if ((inp->inp_vflag & INP_IPV6) != 0) {
 916                                 error = EAFNOSUPPORT;
 917                                 goto out;
 918                         }
 919                         if (IN_MULTICAST(ntohl(sinp->sin_addr.s_addr))) {
 920                                 error = EAFNOSUPPORT;
 921                                 goto out;
 922                         }
 923                         if (ntohl(sinp->sin_addr.s_addr) == INADDR_BROADCAST) {
 924                                 error = EACCES;
 925                                 goto out;
 926                         }
 927                         if ((error = prison_remote_ip4(td->td_ucred,
 928                             &sinp->sin_addr)))
 929                                 goto out;
 930 #ifdef INET6
 931                         isipv6 = 0;
 932 #endif
 933                         break;
 934 #endif /* INET */
 935 #ifdef INET6
 936                 case AF_INET6:
 937                 {
 938                         struct sockaddr_in6 *sin6;
 939
 940                         sin6 = (struct sockaddr_in6 *)nam;
 941                         if (sin6->sin6_len != sizeof(*sin6)) {
 942                                 error = EINVAL;
 943                                 goto out;
 944                         }
 945                         if ((inp->inp_vflag & INP_IPV6PROTO) == 0) {
 946                                 error = EAFNOSUPPORT;
 947                                 goto out;
 948                         }
 949                         if (IN6_IS_ADDR_MULTICAST(&sin6->sin6_addr)) {
 950                                 error = EAFNOSUPPORT;
 951                                 goto out;
 952                         }
 953                         if (IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) {
 954 #ifdef INET
 955                                 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) != 0) {
 956                                         error = EINVAL;
 957                                         goto out;
 958                                 }
 959                                 if ((inp->inp_vflag & INP_IPV4) == 0) {
 960                                         error = EAFNOSUPPORT;
 961                                         goto out;
 962                                 }
 963                                 restoreflags = true;
 964                                 inp->inp_vflag &= ~INP_IPV6;
 965                                 sinp = &sin;
 966                                 in6_sin6_2_sin(sinp, sin6);
 967                                 if (IN_MULTICAST(
 968                                     ntohl(sinp->sin_addr.s_addr))) {
 969                                         error = EAFNOSUPPORT;
 970                                         goto out;
 971                                 }
 972                                 if ((error = prison_remote_ip4(td->td_ucred,
 973                                     &sinp->sin_addr)))
 974                                         goto out;
 975                                 isipv6 = 0;
 976 #else /* !INET */
 977                                 error = EAFNOSUPPORT;
 978                                 goto out;
 979 #endif /* INET */
 980                         } else {
 981                                 if ((inp->inp_vflag & INP_IPV6) == 0) {
 982                                         error = EAFNOSUPPORT;
 983                                         goto out;
 984                                 }
 985                                 restoreflags = true;
 986                                 inp->inp_vflag &= ~INP_IPV4;
 987                                 inp->inp_inc.inc_flags |= INC_ISIPV6;
 988                                 if ((error = prison_remote_ip6(td->td_ucred,
 989                                     &sin6->sin6_addr)))
 990                                         goto out;
 991                                 isipv6 = 1;
 992                         }
 993                         break;
 994                 }
 995 #endif /* INET6 */
 996                 default:
 997                         error = EAFNOSUPPORT;
 998                         goto out;
 999                 }
1000         }
1001         if (!(flags & PRUS_OOB)) {
1002                 if (tp->t_acktime == 0)
1003                         tp->t_acktime = ticks;
1004                 sbappendstream(&so->so_snd, m, flags);
1005                 m = NULL;
1006                 if (nam && tp->t_state < TCPS_SYN_SENT) {
1007                         KASSERT(tp->t_state == TCPS_CLOSED,
1008                             ("%s: tp %p is listening", __func__, tp));
1009
1010                         /*
1011                          * Do implied connect if not yet connected,
1012                          * initialize window to default value, and
1013                          * initialize maxseg using peer's cached MSS.
1014                          */
1015 #ifdef INET6
1016                         if (isipv6)
1017                                 error = tcp6_connect(tp, nam, td);
1018 #endif /* INET6 */
1019 #if defined(INET6) && defined(INET)
1020                         else
1021 #endif
1022 #ifdef INET
1023                                 error = tcp_connect(tp,
1024                                     (struct sockaddr *)sinp, td);
1025 #endif
1026                         /*
1027                          * The bind operation in tcp_connect succeeded. We
1028                          * no longer want to restore the flags if later
1029                          * operations fail.
1030                          */
1031                         if (error == 0 || inp->inp_lport != 0)
1032                                 restoreflags = false;
1033
1034                         if (error) {
1035                                 /* m is freed if PRUS_NOTREADY is unset. */
1036                                 sbflush(&so->so_snd);
1037                                 goto out;
1038                         }
1039                         if (IS_FASTOPEN(tp->t_flags))
1040                                 tcp_fastopen_connect(tp);
1041                         else {
1042                                 tp->snd_wnd = TTCP_CLIENT_SND_WND;
1043                                 tcp_mss(tp, -1);
1044                         }
1045                 }
1046                 if (flags & PRUS_EOF) {
1047                         /*
1048                          * Close the send side of the connection after
1049                          * the data is sent.
1050                          */
1051                         socantsendmore(so);
1052                         tcp_usrclosed(tp);
1053                 }
1054                 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1055                     ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
1056                     (tp->t_fbyte_out == 0) &&
1057                     (so->so_snd.sb_ccc > 0)) {
1058                         tp->t_fbyte_out = ticks;
1059                         if (tp->t_fbyte_out == 0)
1060                                 tp->t_fbyte_out = 1;
1061                         if (tp->t_fbyte_out && tp->t_fbyte_in)
1062                                 tp->t_flags2 |= TF2_FBYTES_COMPLETE;
1063                 }
1064                 if (!(inp->inp_flags & INP_DROPPED) &&
1065                     !(flags & PRUS_NOTREADY)) {
1066                         if (flags & PRUS_MORETOCOME)
1067                                 tp->t_flags |= TF_MORETOCOME;
1068                         error = tcp_output_nodrop(tp);
1069                         if (flags & PRUS_MORETOCOME)
1070                                 tp->t_flags &= ~TF_MORETOCOME;
1071                 }
1072         } else {
1073                 /*
1074                  * XXXRW: PRUS_EOF not implemented with PRUS_OOB?
1075                  */
1076                 SOCKBUF_LOCK(&so->so_snd);
1077                 if (sbspace(&so->so_snd) < -512) {
1078                         SOCKBUF_UNLOCK(&so->so_snd);
1079                         error = ENOBUFS;
1080                         goto out;
1081                 }
1082                 /*
1083                  * According to RFC961 (Assigned Protocols),
1084                  * the urgent pointer points to the last octet
1085                  * of urgent data.  We continue, however,
1086                  * to consider it to indicate the first octet
1087                  * of data past the urgent section.
1088                  * Otherwise, snd_up should be one lower.
1089                  */
1090                 if (tp->t_acktime == 0)
1091                         tp->t_acktime = ticks;
1092                 sbappendstream_locked(&so->so_snd, m, flags);
1093                 SOCKBUF_UNLOCK(&so->so_snd);
1094                 m = NULL;
1095                 if (nam && tp->t_state < TCPS_SYN_SENT) {
1096                         /*
1097                          * Do implied connect if not yet connected,
1098                          * initialize window to default value, and
1099                          * initialize maxseg using peer's cached MSS.
1100                          */
1101
1102                         /*
1103                          * Not going to contemplate SYN|URG
1104                          */
1105                         if (IS_FASTOPEN(tp->t_flags))
1106                                 tp->t_flags &= ~TF_FASTOPEN;
1107 #ifdef INET6
1108                         if (isipv6)
1109                                 error = tcp6_connect(tp, nam, td);
1110 #endif /* INET6 */
1111 #if defined(INET6) && defined(INET)
1112                         else
1113 #endif
1114 #ifdef INET
1115                                 error = tcp_connect(tp,
1116                                     (struct sockaddr *)sinp, td);
1117 #endif
1118                         /*
1119                          * The bind operation in tcp_connect succeeded. We
1120                          * no longer want to restore the flags if later
1121                          * operations fail.
1122                          */
1123                         if (error == 0 || inp->inp_lport != 0)
1124                                 restoreflags = false;
1125
1126                         if (error != 0) {
1127                                 /* m is freed if PRUS_NOTREADY is unset. */
1128                                 sbflush(&so->so_snd);
1129                                 goto out;
1130                         }
1131                         tp->snd_wnd = TTCP_CLIENT_SND_WND;
1132                         tcp_mss(tp, -1);
1133                 }
1134                 tp->snd_up = tp->snd_una + sbavail(&so->so_snd);
1135                 if ((flags & PRUS_NOTREADY) == 0) {
1136                         tp->t_flags |= TF_FORCEDATA;
1137                         error = tcp_output_nodrop(tp);
1138                         tp->t_flags &= ~TF_FORCEDATA;
1139                 }
1140         }
1141         TCP_LOG_EVENT(tp, NULL,
1142             &inp->inp_socket->so_rcv,
1143             &inp->inp_socket->so_snd,
1144             TCP_LOG_USERSEND, error,
1145             0, NULL, false);
1146
1147 out:
1148         /*
1149          * In case of PRUS_NOTREADY, the caller or tcp_usr_ready() is
1150          * responsible for freeing memory.
1151          */
1152         if (m != NULL && (flags & PRUS_NOTREADY) == 0)
1153                 m_freem(m);
1154
1155         /*
1156          * If the request was unsuccessful and we changed flags,
1157          * restore the original flags.
1158          */
1159         if (error != 0 && restoreflags) {
1160                 inp->inp_vflag = vflagsav;
1161                 inp->inp_inc.inc_flags = incflagsav;
1162         }
1163         TCP_PROBE2(debug__user, tp, (flags & PRUS_OOB) ? PRU_SENDOOB :
1164                    ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND));
1165         error = tcp_unlock_or_drop(tp, error);
1166         NET_EPOCH_EXIT(et);
1167         return (error);
1168 }
1169
1170 static int
1171 tcp_usr_ready(struct socket *so, struct mbuf *m, int count)
1172 {
1173         struct epoch_tracker et;
1174         struct inpcb *inp;
1175         struct tcpcb *tp;
1176         int error;
1177
1178         inp = sotoinpcb(so);
1179         INP_WLOCK(inp);
1180         if (inp->inp_flags & INP_DROPPED) {
1181                 INP_WUNLOCK(inp);
1182                 mb_free_notready(m, count);
1183                 return (ECONNRESET);
1184         }
1185         tp = intotcpcb(inp);
1186
1187         SOCKBUF_LOCK(&so->so_snd);
1188         error = sbready(&so->so_snd, m, count);
1189         SOCKBUF_UNLOCK(&so->so_snd);
1190         if (error) {
1191                 INP_WUNLOCK(inp);
1192                 return (error);
1193         }
1194         NET_EPOCH_ENTER(et);
1195         error = tcp_output_unlock(tp);
1196         NET_EPOCH_EXIT(et);
1197
1198         return (error);
1199 }
1200
1201 /*
1202  * Abort the TCP.  Drop the connection abruptly.
1203  */
1204 static void
1205 tcp_usr_abort(struct socket *so)
1206 {
1207         struct inpcb *inp;
1208         struct tcpcb *tp = NULL;
1209         struct epoch_tracker et;
1210
1211         inp = sotoinpcb(so);
1212         KASSERT(inp != NULL, ("tcp_usr_abort: inp == NULL"));
1213
1214         NET_EPOCH_ENTER(et);
1215         INP_WLOCK(inp);
1216         KASSERT(inp->inp_socket != NULL,
1217             ("tcp_usr_abort: inp_socket == NULL"));
1218
1219         /*
1220          * If we still have full TCP state, and we're not dropped, drop.
1221          */
1222         if (!(inp->inp_flags & INP_DROPPED)) {
1223                 tp = intotcpcb(inp);
1224                 tp = tcp_drop(tp, ECONNABORTED);
1225                 if (tp == NULL)
1226                         goto dropped;
1227                 TCP_PROBE2(debug__user, tp, PRU_ABORT);
1228         }
1229         if (!(inp->inp_flags & INP_DROPPED)) {
1230                 soref(so);
1231                 inp->inp_flags |= INP_SOCKREF;
1232         }
1233         INP_WUNLOCK(inp);
1234 dropped:
1235         NET_EPOCH_EXIT(et);
1236 }
1237
1238 /*
1239  * TCP socket is closed.  Start friendly disconnect.
1240  */
1241 static void
1242 tcp_usr_close(struct socket *so)
1243 {
1244         struct inpcb *inp;
1245         struct tcpcb *tp = NULL;
1246         struct epoch_tracker et;
1247
1248         inp = sotoinpcb(so);
1249         KASSERT(inp != NULL, ("tcp_usr_close: inp == NULL"));
1250
1251         NET_EPOCH_ENTER(et);
1252         INP_WLOCK(inp);
1253         KASSERT(inp->inp_socket != NULL,
1254             ("tcp_usr_close: inp_socket == NULL"));
1255
1256         /*
1257          * If we still have full TCP state, and we're not dropped, initiate
1258          * a disconnect.
1259          */
1260         if (!(inp->inp_flags & INP_DROPPED)) {
1261                 tp = intotcpcb(inp);
1262                 tp->t_flags |= TF_CLOSED;
1263                 tcp_disconnect(tp);
1264                 TCP_PROBE2(debug__user, tp, PRU_CLOSE);
1265         }
1266         if (!(inp->inp_flags & INP_DROPPED)) {
1267                 soref(so);
1268                 inp->inp_flags |= INP_SOCKREF;
1269         }
1270         INP_WUNLOCK(inp);
1271         NET_EPOCH_EXIT(et);
1272 }
1273
1274 static int
1275 tcp_pru_options_support(struct tcpcb *tp, int flags)
1276 {
1277         /*
1278          * If the specific TCP stack has a pru_options
1279          * specified then it does not always support
1280          * all the PRU_XX options and we must ask it.
1281          * If the function is not specified then all
1282          * of the PRU_XX options are supported.
1283          */
1284         int ret = 0;
1285
1286         if (tp->t_fb->tfb_pru_options) {
1287                 ret = (*tp->t_fb->tfb_pru_options)(tp, flags);
1288         }
1289         return (ret);
1290 }
1291
1292 /*
1293  * Receive out-of-band data.
1294  */
1295 static int
1296 tcp_usr_rcvoob(struct socket *so, struct mbuf *m, int flags)
1297 {
1298         int error = 0;
1299         struct inpcb *inp;
1300         struct tcpcb *tp = NULL;
1301
1302         inp = sotoinpcb(so);
1303         KASSERT(inp != NULL, ("tcp_usr_rcvoob: inp == NULL"));
1304         INP_WLOCK(inp);
1305         if (inp->inp_flags & INP_DROPPED) {
1306                 error = ECONNRESET;
1307                 goto out;
1308         }
1309         tp = intotcpcb(inp);
1310         error = tcp_pru_options_support(tp, PRUS_OOB);
1311         if (error) {
1312                 goto out;
1313         }
1314         if ((so->so_oobmark == 0 &&
1315              (so->so_rcv.sb_state & SBS_RCVATMARK) == 0) ||
1316             so->so_options & SO_OOBINLINE ||
1317             tp->t_oobflags & TCPOOB_HADDATA) {
1318                 error = EINVAL;
1319                 goto out;
1320         }
1321         if ((tp->t_oobflags & TCPOOB_HAVEDATA) == 0) {
1322                 error = EWOULDBLOCK;
1323                 goto out;
1324         }
1325         m->m_len = 1;
1326         *mtod(m, caddr_t) = tp->t_iobc;
1327         if ((flags & MSG_PEEK) == 0)
1328                 tp->t_oobflags ^= (TCPOOB_HAVEDATA | TCPOOB_HADDATA);
1329
1330 out:
1331         TCP_PROBE2(debug__user, tp, PRU_RCVOOB);
1332         INP_WUNLOCK(inp);
1333         return (error);
1334 }
1335
1336 #ifdef INET
1337 struct protosw tcp_protosw = {
1338         .pr_type =              SOCK_STREAM,
1339         .pr_protocol =          IPPROTO_TCP,
1340         .pr_flags =             PR_CONNREQUIRED | PR_IMPLOPCL | PR_WANTRCVD |
1341                                     PR_CAPATTACH,
1342         .pr_ctloutput =         tcp_ctloutput,
1343         .pr_abort =             tcp_usr_abort,
1344         .pr_accept =            tcp_usr_accept,
1345         .pr_attach =            tcp_usr_attach,
1346         .pr_bind =              tcp_usr_bind,
1347         .pr_connect =           tcp_usr_connect,
1348         .pr_control =           in_control,
1349         .pr_detach =            tcp_usr_detach,
1350         .pr_disconnect =        tcp_usr_disconnect,
1351         .pr_listen =            tcp_usr_listen,
1352         .pr_peeraddr =          in_getpeeraddr,
1353         .pr_rcvd =              tcp_usr_rcvd,
1354         .pr_rcvoob =            tcp_usr_rcvoob,
1355         .pr_send =              tcp_usr_send,
1356         .pr_ready =             tcp_usr_ready,
1357         .pr_shutdown =          tcp_usr_shutdown,
1358         .pr_sockaddr =          in_getsockaddr,
1359         .pr_sosetlabel =        in_pcbsosetlabel,
1360         .pr_close =             tcp_usr_close,
1361 };
1362 #endif /* INET */
1363
1364 #ifdef INET6
1365 struct protosw tcp6_protosw = {
1366         .pr_type =              SOCK_STREAM,
1367         .pr_protocol =          IPPROTO_TCP,
1368         .pr_flags =             PR_CONNREQUIRED | PR_IMPLOPCL |PR_WANTRCVD |
1369                                     PR_CAPATTACH,
1370         .pr_ctloutput =         tcp_ctloutput,
1371         .pr_abort =             tcp_usr_abort,
1372         .pr_accept =            tcp6_usr_accept,
1373         .pr_attach =            tcp_usr_attach,
1374         .pr_bind =              tcp6_usr_bind,
1375         .pr_connect =           tcp6_usr_connect,
1376         .pr_control =           in6_control,
1377         .pr_detach =            tcp_usr_detach,
1378         .pr_disconnect =        tcp_usr_disconnect,
1379         .pr_listen =            tcp6_usr_listen,
1380         .pr_peeraddr =          in6_mapped_peeraddr,
1381         .pr_rcvd =              tcp_usr_rcvd,
1382         .pr_rcvoob =            tcp_usr_rcvoob,
1383         .pr_send =              tcp_usr_send,
1384         .pr_ready =             tcp_usr_ready,
1385         .pr_shutdown =          tcp_usr_shutdown,
1386         .pr_sockaddr =          in6_mapped_sockaddr,
1387         .pr_sosetlabel =        in_pcbsosetlabel,
1388         .pr_close =             tcp_usr_close,
1389 };
1390 #endif /* INET6 */
1391
1392 #ifdef INET
1393 /*
1394  * Common subroutine to open a TCP connection to remote host specified
1395  * by struct sockaddr_in in mbuf *nam.  Call in_pcbbind to assign a local
1396  * port number if needed.  Call in_pcbconnect_setup to do the routing and
1397  * to choose a local host address (interface).  If there is an existing
1398  * incarnation of the same connection in TIME-WAIT state and if the remote
1399  * host was sending CC options and if the connection duration was < MSL, then
1400  * truncate the previous TIME-WAIT state and proceed.
1401  * Initialize connection parameters and enter SYN-SENT state.
1402  */
1403 static int
1404 tcp_connect(struct tcpcb *tp, struct sockaddr *nam, struct thread *td)
1405 {
1406         struct inpcb *inp = tptoinpcb(tp), *oinp;
1407         struct socket *so = tptosocket(tp);
1408         struct in_addr laddr;
1409         u_short lport;
1410         int error;
1411
1412         NET_EPOCH_ASSERT();
1413         INP_WLOCK_ASSERT(inp);
1414
1415         INP_HASH_WLOCK(&V_tcbinfo);
1416         /*
1417          * Cannot simply call in_pcbconnect, because there might be an
1418          * earlier incarnation of this same connection still in
1419          * TIME_WAIT state, creating an ADDRINUSE error.
1420          */
1421         laddr = inp->inp_laddr;
1422         lport = inp->inp_lport;
1423         error = in_pcbconnect_setup(inp, nam, &laddr.s_addr, &lport,
1424             &inp->inp_faddr.s_addr, &inp->inp_fport, &oinp, td->td_ucred);
1425         if (error && oinp == NULL)
1426                 goto out;
1427         if (oinp) {
1428                 error = EADDRINUSE;
1429                 goto out;
1430         }
1431         /* Handle initial bind if it hadn't been done in advance. */
1432         if (inp->inp_lport == 0) {
1433                 inp->inp_lport = lport;
1434                 if (in_pcbinshash(inp) != 0) {
1435                         inp->inp_lport = 0;
1436                         error = EAGAIN;
1437                         goto out;
1438                 }
1439         }
1440         inp->inp_laddr = laddr;
1441         in_pcbrehash(inp);
1442         INP_HASH_WUNLOCK(&V_tcbinfo);
1443
1444         /*
1445          * Compute window scaling to request:
1446          * Scale to fit into sweet spot.  See tcp_syncache.c.
1447          * XXX: This should move to tcp_output().
1448          */
1449         while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
1450             (TCP_MAXWIN << tp->request_r_scale) < sb_max)
1451                 tp->request_r_scale++;
1452
1453         soisconnecting(so);
1454         TCPSTAT_INC(tcps_connattempt);
1455         tcp_state_change(tp, TCPS_SYN_SENT);
1456         tp->iss = tcp_new_isn(&inp->inp_inc);
1457         if (tp->t_flags & TF_REQ_TSTMP)
1458                 tp->ts_offset = tcp_new_ts_offset(&inp->inp_inc);
1459         tcp_sendseqinit(tp);
1460
1461         return 0;
1462
1463 out:
1464         INP_HASH_WUNLOCK(&V_tcbinfo);
1465         return (error);
1466 }
1467 #endif /* INET */
1468
1469 #ifdef INET6
1470 static int
1471 tcp6_connect(struct tcpcb *tp, struct sockaddr *nam, struct thread *td)
1472 {
1473         struct inpcb *inp = tptoinpcb(tp);
1474         struct epoch_tracker et;
1475         int error;
1476
1477         INP_WLOCK_ASSERT(inp);
1478
1479         NET_EPOCH_ENTER(et);
1480         INP_HASH_WLOCK(&V_tcbinfo);
1481         error = in6_pcbconnect(inp, nam, td->td_ucred, true);
1482         INP_HASH_WUNLOCK(&V_tcbinfo);
1483         NET_EPOCH_EXIT(et);
1484         if (error != 0)
1485                 return (error);
1486
1487         /* Compute window scaling to request.  */
1488         while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
1489             (TCP_MAXWIN << tp->request_r_scale) < sb_max)
1490                 tp->request_r_scale++;
1491
1492         soisconnecting(inp->inp_socket);
1493         TCPSTAT_INC(tcps_connattempt);
1494         tcp_state_change(tp, TCPS_SYN_SENT);
1495         tp->iss = tcp_new_isn(&inp->inp_inc);
1496         if (tp->t_flags & TF_REQ_TSTMP)
1497                 tp->ts_offset = tcp_new_ts_offset(&inp->inp_inc);
1498         tcp_sendseqinit(tp);
1499
1500         return (0);
1501 }
1502 #endif /* INET6 */
1503
1504 /*
1505  * Export TCP internal state information via a struct tcp_info, based on the
1506  * Linux 2.6 API.  Not ABI compatible as our constants are mapped differently
1507  * (TCP state machine, etc).  We export all information using FreeBSD-native
1508  * constants -- for example, the numeric values for tcpi_state will differ
1509  * from Linux.
1510  */
1511 static void
1512 tcp_fill_info(struct tcpcb *tp, struct tcp_info *ti)
1513 {
1514
1515         INP_WLOCK_ASSERT(tptoinpcb(tp));
1516         bzero(ti, sizeof(*ti));
1517
1518         ti->tcpi_state = tp->t_state;
1519         if ((tp->t_flags & TF_REQ_TSTMP) && (tp->t_flags & TF_RCVD_TSTMP))
1520                 ti->tcpi_options |= TCPI_OPT_TIMESTAMPS;
1521         if (tp->t_flags & TF_SACK_PERMIT)
1522                 ti->tcpi_options |= TCPI_OPT_SACK;
1523         if ((tp->t_flags & TF_REQ_SCALE) && (tp->t_flags & TF_RCVD_SCALE)) {
1524                 ti->tcpi_options |= TCPI_OPT_WSCALE;
1525                 ti->tcpi_snd_wscale = tp->snd_scale;
1526                 ti->tcpi_rcv_wscale = tp->rcv_scale;
1527         }
1528         if (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))
1529                 ti->tcpi_options |= TCPI_OPT_ECN;
1530
1531         ti->tcpi_rto = tp->t_rxtcur * tick;
1532         ti->tcpi_last_data_recv = ((uint32_t)ticks - tp->t_rcvtime) * tick;
1533         ti->tcpi_rtt = ((u_int64_t)tp->t_srtt * tick) >> TCP_RTT_SHIFT;
1534         ti->tcpi_rttvar = ((u_int64_t)tp->t_rttvar * tick) >> TCP_RTTVAR_SHIFT;
1535
1536         ti->tcpi_snd_ssthresh = tp->snd_ssthresh;
1537         ti->tcpi_snd_cwnd = tp->snd_cwnd;
1538
1539         /*
1540          * FreeBSD-specific extension fields for tcp_info.
1541          */
1542         ti->tcpi_rcv_space = tp->rcv_wnd;
1543         ti->tcpi_rcv_nxt = tp->rcv_nxt;
1544         ti->tcpi_snd_wnd = tp->snd_wnd;
1545         ti->tcpi_snd_bwnd = 0;          /* Unused, kept for compat. */
1546         ti->tcpi_snd_nxt = tp->snd_nxt;
1547         ti->tcpi_snd_mss = tp->t_maxseg;
1548         ti->tcpi_rcv_mss = tp->t_maxseg;
1549         ti->tcpi_snd_rexmitpack = tp->t_sndrexmitpack;
1550         ti->tcpi_rcv_ooopack = tp->t_rcvoopack;
1551         ti->tcpi_snd_zerowin = tp->t_sndzerowin;
1552 #ifdef TCP_OFFLOAD
1553         if (tp->t_flags & TF_TOE) {
1554                 ti->tcpi_options |= TCPI_OPT_TOE;
1555                 tcp_offload_tcp_info(tp, ti);
1556         }
1557 #endif
1558         /*
1559          * AccECN related counters.
1560          */
1561         if ((tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT)) ==
1562             (TF2_ECN_PERMIT | TF2_ACE_PERMIT))
1563                 /*
1564                  * Internal counter starts at 5 for AccECN
1565                  * but 0 for RFC3168 ECN.
1566                  */
1567                 ti->tcpi_delivered_ce = tp->t_scep - 5;
1568         else
1569                 ti->tcpi_delivered_ce = tp->t_scep;
1570         ti->tcpi_received_ce = tp->t_rcep;
1571 }
1572
1573 /*
1574  * tcp_ctloutput() must drop the inpcb lock before performing copyin on
1575  * socket option arguments.  When it re-acquires the lock after the copy, it
1576  * has to revalidate that the connection is still valid for the socket
1577  * option.
1578  */
1579 #define INP_WLOCK_RECHECK_CLEANUP(inp, cleanup) do {                    \
1580         INP_WLOCK(inp);                                                 \
1581         if (inp->inp_flags & INP_DROPPED) {                             \
1582                 INP_WUNLOCK(inp);                                       \
1583                 cleanup;                                                \
1584                 return (ECONNRESET);                                    \
1585         }                                                               \
1586         tp = intotcpcb(inp);                                            \
1587 } while(0)
1588 #define INP_WLOCK_RECHECK(inp) INP_WLOCK_RECHECK_CLEANUP((inp), /* noop */)
1589
1590 int
1591 tcp_ctloutput_set(struct inpcb *inp, struct sockopt *sopt)
1592 {
1593         struct socket *so = inp->inp_socket;
1594         struct tcpcb *tp = intotcpcb(inp);
1595         int error = 0;
1596
1597         MPASS(sopt->sopt_dir == SOPT_SET);
1598         INP_WLOCK_ASSERT(inp);
1599         KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1600             ("inp_flags == %x", inp->inp_flags));
1601         KASSERT(so != NULL, ("inp_socket == NULL"));
1602
1603         if (sopt->sopt_level != IPPROTO_TCP) {
1604                 INP_WUNLOCK(inp);
1605 #ifdef INET6
1606                 if (inp->inp_vflag & INP_IPV6PROTO)
1607                         error = ip6_ctloutput(so, sopt);
1608 #endif
1609 #if defined(INET6) && defined(INET)
1610                 else
1611 #endif
1612 #ifdef INET
1613                         error = ip_ctloutput(so, sopt);
1614 #endif
1615                 /*
1616                  * When an IP-level socket option affects TCP, pass control
1617                  * down to stack tfb_tcp_ctloutput, otherwise return what
1618                  * IP level returned.
1619                  */
1620                 switch (sopt->sopt_level) {
1621 #ifdef INET6
1622                 case IPPROTO_IPV6:
1623                         if ((inp->inp_vflag & INP_IPV6PROTO) == 0)
1624                                 return (error);
1625                         switch (sopt->sopt_name) {
1626                         case IPV6_TCLASS:
1627                                 /* Notify tcp stacks that care (e.g. RACK). */
1628                                 break;
1629                         case IPV6_USE_MIN_MTU:
1630                                 /* Update t_maxseg accordingly. */
1631                                 break;
1632                         default:
1633                                 return (error);
1634                         }
1635                         break;
1636 #endif
1637 #ifdef INET
1638                 case IPPROTO_IP:
1639                         switch (sopt->sopt_name) {
1640                         case IP_TOS:
1641                                 inp->inp_ip_tos &= ~IPTOS_ECN_MASK;
1642                                 break;
1643                         case IP_TTL:
1644                                 /* Notify tcp stacks that care (e.g. RACK). */
1645                                 break;
1646                         default:
1647                                 return (error);
1648                         }
1649                         break;
1650 #endif
1651                 default:
1652                         return (error);
1653                 }
1654                 INP_WLOCK(inp);
1655                 if (inp->inp_flags & INP_DROPPED) {
1656                         INP_WUNLOCK(inp);
1657                         return (ECONNRESET);
1658                 }
1659         } else if (sopt->sopt_name == TCP_FUNCTION_BLK) {
1660                 /*
1661                  * Protect the TCP option TCP_FUNCTION_BLK so
1662                  * that a sub-function can *never* overwrite this.
1663                  */
1664                 struct tcp_function_set fsn;
1665                 struct tcp_function_block *blk;
1666
1667                 INP_WUNLOCK(inp);
1668                 error = sooptcopyin(sopt, &fsn, sizeof fsn, sizeof fsn);
1669                 if (error)
1670                         return (error);
1671
1672                 INP_WLOCK(inp);
1673                 if (inp->inp_flags & INP_DROPPED) {
1674                         INP_WUNLOCK(inp);
1675                         return (ECONNRESET);
1676                 }
1677                 tp = intotcpcb(inp);
1678
1679                 blk = find_and_ref_tcp_functions(&fsn);
1680                 if (blk == NULL) {
1681                         INP_WUNLOCK(inp);
1682                         return (ENOENT);
1683                 }
1684                 if (tp->t_fb == blk) {
1685                         /* You already have this */
1686                         refcount_release(&blk->tfb_refcnt);
1687                         INP_WUNLOCK(inp);
1688                         return (0);
1689                 }
1690                 if (tp->t_state != TCPS_CLOSED) {
1691                         /*
1692                          * The user has advanced the state
1693                          * past the initial point, we may not
1694                          * be able to switch.
1695                          */
1696                         if (blk->tfb_tcp_handoff_ok != NULL) {
1697                                 /*
1698                                  * Does the stack provide a
1699                                  * query mechanism, if so it may
1700                                  * still be possible?
1701                                  */
1702                                 error = (*blk->tfb_tcp_handoff_ok)(tp);
1703                         } else
1704                                 error = EINVAL;
1705                         if (error) {
1706                                 refcount_release(&blk->tfb_refcnt);
1707                                 INP_WUNLOCK(inp);
1708                                 return(error);
1709                         }
1710                 }
1711                 if (blk->tfb_flags & TCP_FUNC_BEING_REMOVED) {
1712                         refcount_release(&blk->tfb_refcnt);
1713                         INP_WUNLOCK(inp);
1714                         return (ENOENT);
1715                 }
1716                 /*
1717                  * Release the old refcnt, the
1718                  * lookup acquired a ref on the
1719                  * new one already.
1720                  */
1721                 if (tp->t_fb->tfb_tcp_fb_fini) {
1722                         struct epoch_tracker et;
1723                         /*
1724                          * Tell the stack to cleanup with 0 i.e.
1725                          * the tcb is not going away.
1726                          */
1727                         NET_EPOCH_ENTER(et);
1728                         (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
1729                         NET_EPOCH_EXIT(et);
1730                 }
1731 #ifdef TCPHPTS
1732                 /* Assure that we are not on any hpts */
1733                 tcp_hpts_remove(tptoinpcb(tp));
1734 #endif
1735                 if (blk->tfb_tcp_fb_init) {
1736                         error = (*blk->tfb_tcp_fb_init)(tp);
1737                         if (error) {
1738                                 refcount_release(&blk->tfb_refcnt);
1739                                 if (tp->t_fb->tfb_tcp_fb_init) {
1740                                         if((*tp->t_fb->tfb_tcp_fb_init)(tp) != 0)  {
1741                                                 /* Fall back failed, drop the connection */
1742                                                 INP_WUNLOCK(inp);
1743                                                 soabort(so);
1744                                                 return (error);
1745                                         }
1746                                 }
1747                                 goto err_out;
1748                         }
1749                 }
1750                 refcount_release(&tp->t_fb->tfb_refcnt);
1751                 tp->t_fb = blk;
1752 #ifdef TCP_OFFLOAD
1753                 if (tp->t_flags & TF_TOE) {
1754                         tcp_offload_ctloutput(tp, sopt->sopt_dir,
1755                              sopt->sopt_name);
1756                 }
1757 #endif
1758 err_out:
1759                 INP_WUNLOCK(inp);
1760                 return (error);
1761         }
1762
1763         /* Pass in the INP locked, callee must unlock it. */
1764         return (tp->t_fb->tfb_tcp_ctloutput(inp, sopt));
1765 }
1766
1767 static int
1768 tcp_ctloutput_get(struct inpcb *inp, struct sockopt *sopt)
1769 {
1770         struct socket *so = inp->inp_socket;
1771         struct tcpcb *tp = intotcpcb(inp);
1772         int error = 0;
1773
1774         MPASS(sopt->sopt_dir == SOPT_GET);
1775         INP_WLOCK_ASSERT(inp);
1776         KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1777             ("inp_flags == %x", inp->inp_flags));
1778         KASSERT(so != NULL, ("inp_socket == NULL"));
1779
1780         if (sopt->sopt_level != IPPROTO_TCP) {
1781                 INP_WUNLOCK(inp);
1782 #ifdef INET6
1783                 if (inp->inp_vflag & INP_IPV6PROTO)
1784                         error = ip6_ctloutput(so, sopt);
1785 #endif /* INET6 */
1786 #if defined(INET6) && defined(INET)
1787                 else
1788 #endif
1789 #ifdef INET
1790                         error = ip_ctloutput(so, sopt);
1791 #endif
1792                 return (error);
1793         }
1794         if (((sopt->sopt_name == TCP_FUNCTION_BLK) ||
1795              (sopt->sopt_name == TCP_FUNCTION_ALIAS))) {
1796                 struct tcp_function_set fsn;
1797
1798                 if (sopt->sopt_name == TCP_FUNCTION_ALIAS) {
1799                         memset(&fsn, 0, sizeof(fsn));
1800                         find_tcp_function_alias(tp->t_fb, &fsn);
1801                 } else {
1802                         strncpy(fsn.function_set_name,
1803                             tp->t_fb->tfb_tcp_block_name,
1804                             TCP_FUNCTION_NAME_LEN_MAX);
1805                         fsn.function_set_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0';
1806                 }
1807                 fsn.pcbcnt = tp->t_fb->tfb_refcnt;
1808                 INP_WUNLOCK(inp);
1809                 error = sooptcopyout(sopt, &fsn, sizeof fsn);
1810                 return (error);
1811         }
1812
1813         /* Pass in the INP locked, callee must unlock it. */
1814         return (tp->t_fb->tfb_tcp_ctloutput(inp, sopt));
1815 }
1816
1817 int
1818 tcp_ctloutput(struct socket *so, struct sockopt *sopt)
1819 {
1820         struct  inpcb *inp;
1821
1822         inp = sotoinpcb(so);
1823         KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL"));
1824
1825         INP_WLOCK(inp);
1826         if (inp->inp_flags & INP_DROPPED) {
1827                 INP_WUNLOCK(inp);
1828                 return (ECONNRESET);
1829         }
1830         if (sopt->sopt_dir == SOPT_SET)
1831                 return (tcp_ctloutput_set(inp, sopt));
1832         else if (sopt->sopt_dir == SOPT_GET)
1833                 return (tcp_ctloutput_get(inp, sopt));
1834         else
1835                 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);
1836 }
1837
1838 /*
1839  * If this assert becomes untrue, we need to change the size of the buf
1840  * variable in tcp_default_ctloutput().
1841  */
1842 #ifdef CTASSERT
1843 CTASSERT(TCP_CA_NAME_MAX <= TCP_LOG_ID_LEN);
1844 CTASSERT(TCP_LOG_REASON_LEN <= TCP_LOG_ID_LEN);
1845 #endif
1846
1847 #ifdef KERN_TLS
1848 static int
1849 copyin_tls_enable(struct sockopt *sopt, struct tls_enable *tls)
1850 {
1851         struct tls_enable_v0 tls_v0;
1852         int error;
1853
1854         if (sopt->sopt_valsize == sizeof(tls_v0)) {
1855                 error = sooptcopyin(sopt, &tls_v0, sizeof(tls_v0),
1856                     sizeof(tls_v0));
1857                 if (error)
1858                         return (error);
1859                 memset(tls, 0, sizeof(*tls));
1860                 tls->cipher_key = tls_v0.cipher_key;
1861                 tls->iv = tls_v0.iv;
1862                 tls->auth_key = tls_v0.auth_key;
1863                 tls->cipher_algorithm = tls_v0.cipher_algorithm;
1864                 tls->cipher_key_len = tls_v0.cipher_key_len;
1865                 tls->iv_len = tls_v0.iv_len;
1866                 tls->auth_algorithm = tls_v0.auth_algorithm;
1867                 tls->auth_key_len = tls_v0.auth_key_len;
1868                 tls->flags = tls_v0.flags;
1869                 tls->tls_vmajor = tls_v0.tls_vmajor;
1870                 tls->tls_vminor = tls_v0.tls_vminor;
1871                 return (0);
1872         }
1873
1874         return (sooptcopyin(sopt, tls, sizeof(*tls), sizeof(*tls)));
1875 }
1876 #endif
1877
1878 extern struct cc_algo newreno_cc_algo;
1879
1880 static int
1881 tcp_set_cc_mod(struct inpcb *inp, struct sockopt *sopt)
1882 {
1883         struct cc_algo *algo;
1884         void *ptr = NULL;
1885         struct tcpcb *tp;
1886         struct cc_var cc_mem;
1887         char    buf[TCP_CA_NAME_MAX];
1888         size_t mem_sz;
1889         int error;
1890
1891         INP_WUNLOCK(inp);
1892         error = sooptcopyin(sopt, buf, TCP_CA_NAME_MAX - 1, 1);
1893         if (error)
1894                 return(error);
1895         buf[sopt->sopt_valsize] = '\0';
1896         CC_LIST_RLOCK();
1897         STAILQ_FOREACH(algo, &cc_list, entries) {
1898                 if (strncmp(buf, algo->name,
1899                             TCP_CA_NAME_MAX) == 0) {
1900                         if (algo->flags & CC_MODULE_BEING_REMOVED) {
1901                                 /* We can't "see" modules being unloaded */
1902                                 continue;
1903                         }
1904                         break;
1905                 }
1906         }
1907         if (algo == NULL) {
1908                 CC_LIST_RUNLOCK();
1909                 return(ESRCH);
1910         }
1911         /*
1912          * With a reference the algorithm cannot be removed
1913          * so we hold a reference through the change process.
1914          */
1915         cc_refer(algo);
1916         CC_LIST_RUNLOCK();
1917         if (algo->cb_init != NULL) {
1918                 /* We can now pre-get the memory for the CC */
1919                 mem_sz = (*algo->cc_data_sz)();
1920                 if (mem_sz == 0) {
1921                         goto no_mem_needed;
1922                 }
1923                 ptr = malloc(mem_sz, M_CC_MEM, M_WAITOK);
1924         } else {
1925 no_mem_needed:
1926                 mem_sz = 0;
1927                 ptr = NULL;
1928         }
1929         /*
1930          * Make sure its all clean and zero and also get
1931          * back the inplock.
1932          */
1933         memset(&cc_mem, 0, sizeof(cc_mem));
1934         INP_WLOCK(inp);
1935         if (inp->inp_flags & INP_DROPPED) {
1936                 INP_WUNLOCK(inp);
1937                 if (ptr)
1938                         free(ptr, M_CC_MEM);
1939                 /* Release our temp reference */
1940                 CC_LIST_RLOCK();
1941                 cc_release(algo);
1942                 CC_LIST_RUNLOCK();
1943                 return (ECONNRESET);
1944         }
1945         tp = intotcpcb(inp);
1946         if (ptr != NULL)
1947                 memset(ptr, 0, mem_sz);
1948         cc_mem.ccvc.tcp = tp;
1949         /*
1950          * We once again hold a write lock over the tcb so it's
1951          * safe to do these things without ordering concerns.
1952          * Note here we init into stack memory.
1953          */
1954         if (algo->cb_init != NULL)
1955                 error = algo->cb_init(&cc_mem, ptr);
1956         else
1957                 error = 0;
1958         /*
1959          * The CC algorithms, when given their memory
1960          * should not fail we could in theory have a
1961          * KASSERT here.
1962          */
1963         if (error == 0) {
1964                 /*
1965                  * Touchdown, lets go ahead and move the
1966                  * connection to the new CC module by
1967                  * copying in the cc_mem after we call
1968                  * the old ones cleanup (if any).
1969                  */
1970                 if (CC_ALGO(tp)->cb_destroy != NULL)
1971                         CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
1972                 /* Detach the old CC from the tcpcb  */
1973                 cc_detach(tp);
1974                 /* Copy in our temp memory that was inited */
1975                 memcpy(&tp->t_ccv, &cc_mem, sizeof(struct cc_var));
1976                 /* Now attach the new, which takes a reference */
1977                 cc_attach(tp, algo);
1978                 /* Ok now are we where we have gotten past any conn_init? */
1979                 if (TCPS_HAVEESTABLISHED(tp->t_state) && (CC_ALGO(tp)->conn_init != NULL)) {
1980                         /* Yep run the connection init for the new CC */
1981                         CC_ALGO(tp)->conn_init(&tp->t_ccv);
1982                 }
1983         } else if (ptr)
1984                 free(ptr, M_CC_MEM);
1985         INP_WUNLOCK(inp);
1986         /* Now lets release our temp reference */
1987         CC_LIST_RLOCK();
1988         cc_release(algo);
1989         CC_LIST_RUNLOCK();
1990         return (error);
1991 }
1992
1993 int
1994 tcp_default_ctloutput(struct inpcb *inp, struct sockopt *sopt)
1995 {
1996         struct tcpcb *tp = intotcpcb(inp);
1997         int     error, opt, optval;
1998         u_int   ui;
1999         struct  tcp_info ti;
2000 #ifdef KERN_TLS
2001         struct tls_enable tls;
2002         struct socket *so = inp->inp_socket;
2003 #endif
2004         char    *pbuf, buf[TCP_LOG_ID_LEN];
2005 #ifdef STATS
2006         struct statsblob *sbp;
2007 #endif
2008         size_t  len;
2009
2010         INP_WLOCK_ASSERT(inp);
2011         KASSERT((inp->inp_flags & INP_DROPPED) == 0,
2012             ("inp_flags == %x", inp->inp_flags));
2013         KASSERT(inp->inp_socket != NULL, ("inp_socket == NULL"));
2014
2015         switch (sopt->sopt_level) {
2016 #ifdef INET6
2017         case IPPROTO_IPV6:
2018                 MPASS(inp->inp_vflag & INP_IPV6PROTO);
2019                 switch (sopt->sopt_name) {
2020                 case IPV6_USE_MIN_MTU:
2021                         tcp6_use_min_mtu(tp);
2022                         /* FALLTHROUGH */
2023                 }
2024                 INP_WUNLOCK(inp);
2025                 return (0);
2026 #endif
2027 #ifdef INET
2028         case IPPROTO_IP:
2029                 INP_WUNLOCK(inp);
2030                 return (0);
2031 #endif
2032         }
2033
2034         /*
2035          * For TCP_CCALGOOPT forward the control to CC module, for both
2036          * SOPT_SET and SOPT_GET.
2037          */
2038         switch (sopt->sopt_name) {
2039         case TCP_CCALGOOPT:
2040                 INP_WUNLOCK(inp);
2041                 if (sopt->sopt_valsize > CC_ALGOOPT_LIMIT)
2042                         return (EINVAL);
2043                 pbuf = malloc(sopt->sopt_valsize, M_TEMP, M_WAITOK | M_ZERO);
2044                 error = sooptcopyin(sopt, pbuf, sopt->sopt_valsize,
2045                     sopt->sopt_valsize);
2046                 if (error) {
2047                         free(pbuf, M_TEMP);
2048                         return (error);
2049                 }
2050                 INP_WLOCK_RECHECK_CLEANUP(inp, free(pbuf, M_TEMP));
2051                 if (CC_ALGO(tp)->ctl_output != NULL)
2052                         error = CC_ALGO(tp)->ctl_output(&tp->t_ccv, sopt, pbuf);
2053                 else
2054                         error = ENOENT;
2055                 INP_WUNLOCK(inp);
2056                 if (error == 0 && sopt->sopt_dir == SOPT_GET)
2057                         error = sooptcopyout(sopt, pbuf, sopt->sopt_valsize);
2058                 free(pbuf, M_TEMP);
2059                 return (error);
2060         }
2061
2062         switch (sopt->sopt_dir) {
2063         case SOPT_SET:
2064                 switch (sopt->sopt_name) {
2065 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2066                 case TCP_MD5SIG:
2067                         INP_WUNLOCK(inp);
2068                         if (!TCPMD5_ENABLED())
2069                                 return (ENOPROTOOPT);
2070                         error = TCPMD5_PCBCTL(inp, sopt);
2071                         if (error)
2072                                 return (error);
2073                         INP_WLOCK_RECHECK(inp);
2074                         goto unlock_and_done;
2075 #endif /* IPSEC */
2076
2077                 case TCP_NODELAY:
2078                 case TCP_NOOPT:
2079                 case TCP_LRD:
2080                         INP_WUNLOCK(inp);
2081                         error = sooptcopyin(sopt, &optval, sizeof optval,
2082                             sizeof optval);
2083                         if (error)
2084                                 return (error);
2085
2086                         INP_WLOCK_RECHECK(inp);
2087                         switch (sopt->sopt_name) {
2088                         case TCP_NODELAY:
2089                                 opt = TF_NODELAY;
2090                                 break;
2091                         case TCP_NOOPT:
2092                                 opt = TF_NOOPT;
2093                                 break;
2094                         case TCP_LRD:
2095                                 opt = TF_LRD;
2096                                 break;
2097                         default:
2098                                 opt = 0; /* dead code to fool gcc */
2099                                 break;
2100                         }
2101
2102                         if (optval)
2103                                 tp->t_flags |= opt;
2104                         else
2105                                 tp->t_flags &= ~opt;
2106 unlock_and_done:
2107 #ifdef TCP_OFFLOAD
2108                         if (tp->t_flags & TF_TOE) {
2109                                 tcp_offload_ctloutput(tp, sopt->sopt_dir,
2110                                     sopt->sopt_name);
2111                         }
2112 #endif
2113                         INP_WUNLOCK(inp);
2114                         break;
2115
2116                 case TCP_NOPUSH:
2117                         INP_WUNLOCK(inp);
2118                         error = sooptcopyin(sopt, &optval, sizeof optval,
2119                             sizeof optval);
2120                         if (error)
2121                                 return (error);
2122
2123                         INP_WLOCK_RECHECK(inp);
2124                         if (optval)
2125                                 tp->t_flags |= TF_NOPUSH;
2126                         else if (tp->t_flags & TF_NOPUSH) {
2127                                 tp->t_flags &= ~TF_NOPUSH;
2128                                 if (TCPS_HAVEESTABLISHED(tp->t_state)) {
2129                                         struct epoch_tracker et;
2130
2131                                         NET_EPOCH_ENTER(et);
2132                                         error = tcp_output_nodrop(tp);
2133                                         NET_EPOCH_EXIT(et);
2134                                 }
2135                         }
2136                         goto unlock_and_done;
2137
2138                 case TCP_REMOTE_UDP_ENCAPS_PORT:
2139                         INP_WUNLOCK(inp);
2140                         error = sooptcopyin(sopt, &optval, sizeof optval,
2141                             sizeof optval);
2142                         if (error)
2143                                 return (error);
2144                         if ((optval < TCP_TUNNELING_PORT_MIN) ||
2145                             (optval > TCP_TUNNELING_PORT_MAX)) {
2146                                 /* Its got to be in range */
2147                                 return (EINVAL);
2148                         }
2149                         if ((V_tcp_udp_tunneling_port == 0) && (optval != 0)) {
2150                                 /* You have to have enabled a UDP tunneling port first */
2151                                 return (EINVAL);
2152                         }
2153                         INP_WLOCK_RECHECK(inp);
2154                         if (tp->t_state != TCPS_CLOSED) {
2155                                 /* You can't change after you are connected */
2156                                 error = EINVAL;
2157                         } else {
2158                                 /* Ok we are all good set the port */
2159                                 tp->t_port = htons(optval);
2160                         }
2161                         goto unlock_and_done;
2162
2163                 case TCP_MAXSEG:
2164                         INP_WUNLOCK(inp);
2165                         error = sooptcopyin(sopt, &optval, sizeof optval,
2166                             sizeof optval);
2167                         if (error)
2168                                 return (error);
2169
2170                         INP_WLOCK_RECHECK(inp);
2171                         if (optval > 0 && optval <= tp->t_maxseg &&
2172                             optval + 40 >= V_tcp_minmss)
2173                                 tp->t_maxseg = optval;
2174                         else
2175                                 error = EINVAL;
2176                         goto unlock_and_done;
2177
2178                 case TCP_INFO:
2179                         INP_WUNLOCK(inp);
2180                         error = EINVAL;
2181                         break;
2182
2183                 case TCP_STATS:
2184                         INP_WUNLOCK(inp);
2185 #ifdef STATS
2186                         error = sooptcopyin(sopt, &optval, sizeof optval,
2187                             sizeof optval);
2188                         if (error)
2189                                 return (error);
2190
2191                         if (optval > 0)
2192                                 sbp = stats_blob_alloc(
2193                                     V_tcp_perconn_stats_dflt_tpl, 0);
2194                         else
2195                                 sbp = NULL;
2196
2197                         INP_WLOCK_RECHECK(inp);
2198                         if ((tp->t_stats != NULL && sbp == NULL) ||
2199                             (tp->t_stats == NULL && sbp != NULL)) {
2200                                 struct statsblob *t = tp->t_stats;
2201                                 tp->t_stats = sbp;
2202                                 sbp = t;
2203                         }
2204                         INP_WUNLOCK(inp);
2205
2206                         stats_blob_destroy(sbp);
2207 #else
2208                         return (EOPNOTSUPP);
2209 #endif /* !STATS */
2210                         break;
2211
2212                 case TCP_CONGESTION:
2213                         error = tcp_set_cc_mod(inp, sopt);
2214                         break;
2215
2216                 case TCP_REUSPORT_LB_NUMA:
2217                         INP_WUNLOCK(inp);
2218                         error = sooptcopyin(sopt, &optval, sizeof(optval),
2219                             sizeof(optval));
2220                         INP_WLOCK_RECHECK(inp);
2221                         if (!error)
2222                                 error = in_pcblbgroup_numa(inp, optval);
2223                         INP_WUNLOCK(inp);
2224                         break;
2225
2226 #ifdef KERN_TLS
2227                 case TCP_TXTLS_ENABLE:
2228                         INP_WUNLOCK(inp);
2229                         error = copyin_tls_enable(sopt, &tls);
2230                         if (error)
2231                                 break;
2232                         error = ktls_enable_tx(so, &tls);
2233                         break;
2234                 case TCP_TXTLS_MODE:
2235                         INP_WUNLOCK(inp);
2236                         error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
2237                         if (error)
2238                                 return (error);
2239
2240                         INP_WLOCK_RECHECK(inp);
2241                         error = ktls_set_tx_mode(so, ui);
2242                         INP_WUNLOCK(inp);
2243                         break;
2244                 case TCP_RXTLS_ENABLE:
2245                         INP_WUNLOCK(inp);
2246                         error = sooptcopyin(sopt, &tls, sizeof(tls),
2247                             sizeof(tls));
2248                         if (error)
2249                                 break;
2250                         error = ktls_enable_rx(so, &tls);
2251                         break;
2252 #endif
2253                 case TCP_MAXUNACKTIME:
2254                 case TCP_KEEPIDLE:
2255                 case TCP_KEEPINTVL:
2256                 case TCP_KEEPINIT:
2257                         INP_WUNLOCK(inp);
2258                         error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
2259                         if (error)
2260                                 return (error);
2261
2262                         if (ui > (UINT_MAX / hz)) {
2263                                 error = EINVAL;
2264                                 break;
2265                         }
2266                         ui *= hz;
2267
2268                         INP_WLOCK_RECHECK(inp);
2269                         switch (sopt->sopt_name) {
2270                         case TCP_MAXUNACKTIME:
2271                                 tp->t_maxunacktime = ui;
2272                                 break;
2273
2274                         case TCP_KEEPIDLE:
2275                                 tp->t_keepidle = ui;
2276                                 /*
2277                                  * XXX: better check current remaining
2278                                  * timeout and "merge" it with new value.
2279                                  */
2280                                 if ((tp->t_state > TCPS_LISTEN) &&
2281                                     (tp->t_state <= TCPS_CLOSING))
2282                                         tcp_timer_activate(tp, TT_KEEP,
2283                                             TP_KEEPIDLE(tp));
2284                                 break;
2285                         case TCP_KEEPINTVL:
2286                                 tp->t_keepintvl = ui;
2287                                 if ((tp->t_state == TCPS_FIN_WAIT_2) &&
2288                                     (TP_MAXIDLE(tp) > 0))
2289                                         tcp_timer_activate(tp, TT_2MSL,
2290                                             TP_MAXIDLE(tp));
2291                                 break;
2292                         case TCP_KEEPINIT:
2293                                 tp->t_keepinit = ui;
2294                                 if (tp->t_state == TCPS_SYN_RECEIVED ||
2295                                     tp->t_state == TCPS_SYN_SENT)
2296                                         tcp_timer_activate(tp, TT_KEEP,
2297                                             TP_KEEPINIT(tp));
2298                                 break;
2299                         }
2300                         goto unlock_and_done;
2301
2302                 case TCP_KEEPCNT:
2303                         INP_WUNLOCK(inp);
2304                         error = sooptcopyin(sopt, &ui, sizeof(ui), sizeof(ui));
2305                         if (error)
2306                                 return (error);
2307
2308                         INP_WLOCK_RECHECK(inp);
2309                         tp->t_keepcnt = ui;
2310                         if ((tp->t_state == TCPS_FIN_WAIT_2) &&
2311                             (TP_MAXIDLE(tp) > 0))
2312                                 tcp_timer_activate(tp, TT_2MSL,
2313                                     TP_MAXIDLE(tp));
2314                         goto unlock_and_done;
2315
2316 #ifdef TCPPCAP
2317                 case TCP_PCAP_OUT:
2318                 case TCP_PCAP_IN:
2319                         INP_WUNLOCK(inp);
2320                         error = sooptcopyin(sopt, &optval, sizeof optval,
2321                             sizeof optval);
2322                         if (error)
2323                                 return (error);
2324
2325                         INP_WLOCK_RECHECK(inp);
2326                         if (optval >= 0)
2327                                 tcp_pcap_set_sock_max(TCP_PCAP_OUT ?
2328                                         &(tp->t_outpkts) : &(tp->t_inpkts),
2329                                         optval);
2330                         else
2331                                 error = EINVAL;
2332                         goto unlock_and_done;
2333 #endif
2334
2335                 case TCP_FASTOPEN: {
2336                         struct tcp_fastopen tfo_optval;
2337
2338                         INP_WUNLOCK(inp);
2339                         if (!V_tcp_fastopen_client_enable &&
2340                             !V_tcp_fastopen_server_enable)
2341                                 return (EPERM);
2342
2343                         error = sooptcopyin(sopt, &tfo_optval,
2344                                     sizeof(tfo_optval), sizeof(int));
2345                         if (error)
2346                                 return (error);
2347
2348                         INP_WLOCK_RECHECK(inp);
2349                         if ((tp->t_state != TCPS_CLOSED) &&
2350                             (tp->t_state != TCPS_LISTEN)) {
2351                                 error = EINVAL;
2352                                 goto unlock_and_done;
2353                         }
2354                         if (tfo_optval.enable) {
2355                                 if (tp->t_state == TCPS_LISTEN) {
2356                                         if (!V_tcp_fastopen_server_enable) {
2357                                                 error = EPERM;
2358                                                 goto unlock_and_done;
2359                                         }
2360
2361                                         if (tp->t_tfo_pending == NULL)
2362                                                 tp->t_tfo_pending =
2363                                                     tcp_fastopen_alloc_counter();
2364                                 } else {
2365                                         /*
2366                                          * If a pre-shared key was provided,
2367                                          * stash it in the client cookie
2368                                          * field of the tcpcb for use during
2369                                          * connect.
2370                                          */
2371                                         if (sopt->sopt_valsize ==
2372                                             sizeof(tfo_optval)) {
2373                                                 memcpy(tp->t_tfo_cookie.client,
2374                                                        tfo_optval.psk,
2375                                                        TCP_FASTOPEN_PSK_LEN);
2376                                                 tp->t_tfo_client_cookie_len =
2377                                                     TCP_FASTOPEN_PSK_LEN;
2378                                         }
2379                                 }
2380                                 tp->t_flags |= TF_FASTOPEN;
2381                         } else
2382                                 tp->t_flags &= ~TF_FASTOPEN;
2383                         goto unlock_and_done;
2384                 }
2385
2386 #ifdef TCP_BLACKBOX
2387                 case TCP_LOG:
2388                         INP_WUNLOCK(inp);
2389                         error = sooptcopyin(sopt, &optval, sizeof optval,
2390                             sizeof optval);
2391                         if (error)
2392                                 return (error);
2393
2394                         INP_WLOCK_RECHECK(inp);
2395                         error = tcp_log_state_change(tp, optval);
2396                         goto unlock_and_done;
2397
2398                 case TCP_LOGBUF:
2399                         INP_WUNLOCK(inp);
2400                         error = EINVAL;
2401                         break;
2402
2403                 case TCP_LOGID:
2404                         INP_WUNLOCK(inp);
2405                         error = sooptcopyin(sopt, buf, TCP_LOG_ID_LEN - 1, 0);
2406                         if (error)
2407                                 break;
2408                         buf[sopt->sopt_valsize] = '\0';
2409                         INP_WLOCK_RECHECK(inp);
2410                         error = tcp_log_set_id(tp, buf);
2411                         /* tcp_log_set_id() unlocks the INP. */
2412                         break;
2413
2414                 case TCP_LOGDUMP:
2415                 case TCP_LOGDUMPID:
2416                         INP_WUNLOCK(inp);
2417                         error =
2418                             sooptcopyin(sopt, buf, TCP_LOG_REASON_LEN - 1, 0);
2419                         if (error)
2420                                 break;
2421                         buf[sopt->sopt_valsize] = '\0';
2422                         INP_WLOCK_RECHECK(inp);
2423                         if (sopt->sopt_name == TCP_LOGDUMP) {
2424                                 error = tcp_log_dump_tp_logbuf(tp, buf,
2425                                     M_WAITOK, true);
2426                                 INP_WUNLOCK(inp);
2427                         } else {
2428                                 tcp_log_dump_tp_bucket_logbufs(tp, buf);
2429                                 /*
2430                                  * tcp_log_dump_tp_bucket_logbufs() drops the
2431                                  * INP lock.
2432                                  */
2433                         }
2434                         break;
2435 #endif
2436
2437                 default:
2438                         INP_WUNLOCK(inp);
2439                         error = ENOPROTOOPT;
2440                         break;
2441                 }
2442                 break;
2443
2444         case SOPT_GET:
2445                 tp = intotcpcb(inp);
2446                 switch (sopt->sopt_name) {
2447 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2448                 case TCP_MD5SIG:
2449                         INP_WUNLOCK(inp);
2450                         if (!TCPMD5_ENABLED())
2451                                 return (ENOPROTOOPT);
2452                         error = TCPMD5_PCBCTL(inp, sopt);
2453                         break;
2454 #endif
2455
2456                 case TCP_NODELAY:
2457                         optval = tp->t_flags & TF_NODELAY;
2458                         INP_WUNLOCK(inp);
2459                         error = sooptcopyout(sopt, &optval, sizeof optval);
2460                         break;
2461                 case TCP_MAXSEG:
2462                         optval = tp->t_maxseg;
2463                         INP_WUNLOCK(inp);
2464                         error = sooptcopyout(sopt, &optval, sizeof optval);
2465                         break;
2466                 case TCP_REMOTE_UDP_ENCAPS_PORT:
2467                         optval = ntohs(tp->t_port);
2468                         INP_WUNLOCK(inp);
2469                         error = sooptcopyout(sopt, &optval, sizeof optval);
2470                         break;
2471                 case TCP_NOOPT:
2472                         optval = tp->t_flags & TF_NOOPT;
2473                         INP_WUNLOCK(inp);
2474                         error = sooptcopyout(sopt, &optval, sizeof optval);
2475                         break;
2476                 case TCP_NOPUSH:
2477                         optval = tp->t_flags & TF_NOPUSH;
2478                         INP_WUNLOCK(inp);
2479                         error = sooptcopyout(sopt, &optval, sizeof optval);
2480                         break;
2481                 case TCP_INFO:
2482                         tcp_fill_info(tp, &ti);
2483                         INP_WUNLOCK(inp);
2484                         error = sooptcopyout(sopt, &ti, sizeof ti);
2485                         break;
2486                 case TCP_STATS:
2487                         {
2488 #ifdef STATS
2489                         int nheld;
2490                         TYPEOF_MEMBER(struct statsblob, flags) sbflags = 0;
2491
2492                         error = 0;
2493                         socklen_t outsbsz = sopt->sopt_valsize;
2494                         if (tp->t_stats == NULL)
2495                                 error = ENOENT;
2496                         else if (outsbsz >= tp->t_stats->cursz)
2497                                 outsbsz = tp->t_stats->cursz;
2498                         else if (outsbsz >= sizeof(struct statsblob))
2499                                 outsbsz = sizeof(struct statsblob);
2500                         else
2501                                 error = EINVAL;
2502                         INP_WUNLOCK(inp);
2503                         if (error)
2504                                 break;
2505
2506                         sbp = sopt->sopt_val;
2507                         nheld = atop(round_page(((vm_offset_t)sbp) +
2508                             (vm_size_t)outsbsz) - trunc_page((vm_offset_t)sbp));
2509                         vm_page_t ma[nheld];
2510                         if (vm_fault_quick_hold_pages(
2511                             &curproc->p_vmspace->vm_map, (vm_offset_t)sbp,
2512                             outsbsz, VM_PROT_READ | VM_PROT_WRITE, ma,
2513                             nheld) < 0) {
2514                                 error = EFAULT;
2515                                 break;
2516                         }
2517
2518                         if ((error = copyin_nofault(&(sbp->flags), &sbflags,
2519                             SIZEOF_MEMBER(struct statsblob, flags))))
2520                                 goto unhold;
2521
2522                         INP_WLOCK_RECHECK(inp);
2523                         error = stats_blob_snapshot(&sbp, outsbsz, tp->t_stats,
2524                             sbflags | SB_CLONE_USRDSTNOFAULT);
2525                         INP_WUNLOCK(inp);
2526                         sopt->sopt_valsize = outsbsz;
2527 unhold:
2528                         vm_page_unhold_pages(ma, nheld);
2529 #else
2530                         INP_WUNLOCK(inp);
2531                         error = EOPNOTSUPP;
2532 #endif /* !STATS */
2533                         break;
2534                         }
2535                 case TCP_CONGESTION:
2536                         len = strlcpy(buf, CC_ALGO(tp)->name, TCP_CA_NAME_MAX);
2537                         INP_WUNLOCK(inp);
2538                         error = sooptcopyout(sopt, buf, len + 1);
2539                         break;
2540                 case TCP_MAXUNACKTIME:
2541                 case TCP_KEEPIDLE:
2542                 case TCP_KEEPINTVL:
2543                 case TCP_KEEPINIT:
2544                 case TCP_KEEPCNT:
2545                         switch (sopt->sopt_name) {
2546                         case TCP_MAXUNACKTIME:
2547                                 ui = TP_MAXUNACKTIME(tp) / hz;
2548                                 break;
2549                         case TCP_KEEPIDLE:
2550                                 ui = TP_KEEPIDLE(tp) / hz;
2551                                 break;
2552                         case TCP_KEEPINTVL:
2553                                 ui = TP_KEEPINTVL(tp) / hz;
2554                                 break;
2555                         case TCP_KEEPINIT:
2556                                 ui = TP_KEEPINIT(tp) / hz;
2557                                 break;
2558                         case TCP_KEEPCNT:
2559                                 ui = TP_KEEPCNT(tp);
2560                                 break;
2561                         }
2562                         INP_WUNLOCK(inp);
2563                         error = sooptcopyout(sopt, &ui, sizeof(ui));
2564                         break;
2565 #ifdef TCPPCAP
2566                 case TCP_PCAP_OUT:
2567                 case TCP_PCAP_IN:
2568                         optval = tcp_pcap_get_sock_max(TCP_PCAP_OUT ?
2569                                         &(tp->t_outpkts) : &(tp->t_inpkts));
2570                         INP_WUNLOCK(inp);
2571                         error = sooptcopyout(sopt, &optval, sizeof optval);
2572                         break;
2573 #endif
2574                 case TCP_FASTOPEN:
2575                         optval = tp->t_flags & TF_FASTOPEN;
2576                         INP_WUNLOCK(inp);
2577                         error = sooptcopyout(sopt, &optval, sizeof optval);
2578                         break;
2579 #ifdef TCP_BLACKBOX
2580                 case TCP_LOG:
2581                         optval = tp->t_logstate;
2582                         INP_WUNLOCK(inp);
2583                         error = sooptcopyout(sopt, &optval, sizeof(optval));
2584                         break;
2585                 case TCP_LOGBUF:
2586                         /* tcp_log_getlogbuf() does INP_WUNLOCK(inp) */
2587                         error = tcp_log_getlogbuf(sopt, tp);
2588                         break;
2589                 case TCP_LOGID:
2590                         len = tcp_log_get_id(tp, buf);
2591                         INP_WUNLOCK(inp);
2592                         error = sooptcopyout(sopt, buf, len + 1);
2593                         break;
2594                 case TCP_LOGDUMP:
2595                 case TCP_LOGDUMPID:
2596                         INP_WUNLOCK(inp);
2597                         error = EINVAL;
2598                         break;
2599 #endif
2600 #ifdef KERN_TLS
2601                 case TCP_TXTLS_MODE:
2602                         error = ktls_get_tx_mode(so, &optval);
2603                         INP_WUNLOCK(inp);
2604                         if (error == 0)
2605                                 error = sooptcopyout(sopt, &optval,
2606                                     sizeof(optval));
2607                         break;
2608                 case TCP_RXTLS_MODE:
2609                         error = ktls_get_rx_mode(so, &optval);
2610                         INP_WUNLOCK(inp);
2611                         if (error == 0)
2612                                 error = sooptcopyout(sopt, &optval,
2613                                     sizeof(optval));
2614                         break;
2615 #endif
2616                 case TCP_LRD:
2617                         optval = tp->t_flags & TF_LRD;
2618                         INP_WUNLOCK(inp);
2619                         error = sooptcopyout(sopt, &optval, sizeof optval);
2620                         break;
2621                 default:
2622                         INP_WUNLOCK(inp);
2623                         error = ENOPROTOOPT;
2624                         break;
2625                 }
2626                 break;
2627         }
2628         return (error);
2629 }
2630 #undef INP_WLOCK_RECHECK
2631 #undef INP_WLOCK_RECHECK_CLEANUP
2632
2633 /*
2634  * Initiate (or continue) disconnect.
2635  * If embryonic state, just send reset (once).
2636  * If in ``let data drain'' option and linger null, just drop.
2637  * Otherwise (hard), mark socket disconnecting and drop
2638  * current input data; switch states based on user close, and
2639  * send segment to peer (with FIN).
2640  */
2641 static void
2642 tcp_disconnect(struct tcpcb *tp)
2643 {
2644         struct inpcb *inp = tptoinpcb(tp);
2645         struct socket *so = tptosocket(tp);
2646
2647         NET_EPOCH_ASSERT();
2648         INP_WLOCK_ASSERT(inp);
2649
2650         /*
2651          * Neither tcp_close() nor tcp_drop() should return NULL, as the
2652          * socket is still open.
2653          */
2654         if (tp->t_state < TCPS_ESTABLISHED &&
2655             !(tp->t_state > TCPS_LISTEN && IS_FASTOPEN(tp->t_flags))) {
2656                 tp = tcp_close(tp);
2657                 KASSERT(tp != NULL,
2658                     ("tcp_disconnect: tcp_close() returned NULL"));
2659         } else if ((so->so_options & SO_LINGER) && so->so_linger == 0) {
2660                 tp = tcp_drop(tp, 0);
2661                 KASSERT(tp != NULL,
2662                     ("tcp_disconnect: tcp_drop() returned NULL"));
2663         } else {
2664                 soisdisconnecting(so);
2665                 sbflush(&so->so_rcv);
2666                 tcp_usrclosed(tp);
2667                 if (!(inp->inp_flags & INP_DROPPED))
2668                         /* Ignore stack's drop request, we already at it. */
2669                         (void)tcp_output_nodrop(tp);
2670         }
2671 }
2672
2673 /*
2674  * User issued close, and wish to trail through shutdown states:
2675  * if never received SYN, just forget it.  If got a SYN from peer,
2676  * but haven't sent FIN, then go to FIN_WAIT_1 state to send peer a FIN.
2677  * If already got a FIN from peer, then almost done; go to LAST_ACK
2678  * state.  In all other cases, have already sent FIN to peer (e.g.
2679  * after PRU_SHUTDOWN), and just have to play tedious game waiting
2680  * for peer to send FIN or not respond to keep-alives, etc.
2681  * We can let the user exit from the close as soon as the FIN is acked.
2682  */
2683 static void
2684 tcp_usrclosed(struct tcpcb *tp)
2685 {
2686
2687         NET_EPOCH_ASSERT();
2688         INP_WLOCK_ASSERT(tptoinpcb(tp));
2689
2690         switch (tp->t_state) {
2691         case TCPS_LISTEN:
2692 #ifdef TCP_OFFLOAD
2693                 tcp_offload_listen_stop(tp);
2694 #endif
2695                 tcp_state_change(tp, TCPS_CLOSED);
2696                 /* FALLTHROUGH */
2697         case TCPS_CLOSED:
2698                 tp = tcp_close(tp);
2699                 /*
2700                  * tcp_close() should never return NULL here as the socket is
2701                  * still open.
2702                  */
2703                 KASSERT(tp != NULL,
2704                     ("tcp_usrclosed: tcp_close() returned NULL"));
2705                 break;
2706
2707         case TCPS_SYN_SENT:
2708         case TCPS_SYN_RECEIVED:
2709                 tp->t_flags |= TF_NEEDFIN;
2710                 break;
2711
2712         case TCPS_ESTABLISHED:
2713                 tcp_state_change(tp, TCPS_FIN_WAIT_1);
2714                 break;
2715
2716         case TCPS_CLOSE_WAIT:
2717                 tcp_state_change(tp, TCPS_LAST_ACK);
2718                 break;
2719         }
2720         if (tp->t_acktime == 0)
2721                 tp->t_acktime = ticks;
2722         if (tp->t_state >= TCPS_FIN_WAIT_2) {
2723                 soisdisconnected(tptosocket(tp));
2724                 /* Prevent the connection hanging in FIN_WAIT_2 forever. */
2725                 if (tp->t_state == TCPS_FIN_WAIT_2) {
2726                         int timeout;
2727
2728                         timeout = (tcp_fast_finwait2_recycle) ?
2729                             tcp_finwait2_timeout : TP_MAXIDLE(tp);
2730                         tcp_timer_activate(tp, TT_2MSL, timeout);
2731                 }
2732         }
2733 }
2734
2735 #ifdef DDB
2736 static void
2737 db_print_indent(int indent)
2738 {
2739         int i;
2740
2741         for (i = 0; i < indent; i++)
2742                 db_printf(" ");
2743 }
2744
2745 static void
2746 db_print_tstate(int t_state)
2747 {
2748
2749         switch (t_state) {
2750         case TCPS_CLOSED:
2751                 db_printf("TCPS_CLOSED");
2752                 return;
2753
2754         case TCPS_LISTEN:
2755                 db_printf("TCPS_LISTEN");
2756                 return;
2757
2758         case TCPS_SYN_SENT:
2759                 db_printf("TCPS_SYN_SENT");
2760                 return;
2761
2762         case TCPS_SYN_RECEIVED:
2763                 db_printf("TCPS_SYN_RECEIVED");
2764                 return;
2765
2766         case TCPS_ESTABLISHED:
2767                 db_printf("TCPS_ESTABLISHED");
2768                 return;
2769
2770         case TCPS_CLOSE_WAIT:
2771                 db_printf("TCPS_CLOSE_WAIT");
2772                 return;
2773
2774         case TCPS_FIN_WAIT_1:
2775                 db_printf("TCPS_FIN_WAIT_1");
2776                 return;
2777
2778         case TCPS_CLOSING:
2779                 db_printf("TCPS_CLOSING");
2780                 return;
2781
2782         case TCPS_LAST_ACK:
2783                 db_printf("TCPS_LAST_ACK");
2784                 return;
2785
2786         case TCPS_FIN_WAIT_2:
2787                 db_printf("TCPS_FIN_WAIT_2");
2788                 return;
2789
2790         case TCPS_TIME_WAIT:
2791                 db_printf("TCPS_TIME_WAIT");
2792                 return;
2793
2794         default:
2795                 db_printf("unknown");
2796                 return;
2797         }
2798 }
2799
2800 static void
2801 db_print_tflags(u_int t_flags)
2802 {
2803         int comma;
2804
2805         comma = 0;
2806         if (t_flags & TF_ACKNOW) {
2807                 db_printf("%sTF_ACKNOW", comma ? ", " : "");
2808                 comma = 1;
2809         }
2810         if (t_flags & TF_DELACK) {
2811                 db_printf("%sTF_DELACK", comma ? ", " : "");
2812                 comma = 1;
2813         }
2814         if (t_flags & TF_NODELAY) {
2815                 db_printf("%sTF_NODELAY", comma ? ", " : "");
2816                 comma = 1;
2817         }
2818         if (t_flags & TF_NOOPT) {
2819                 db_printf("%sTF_NOOPT", comma ? ", " : "");
2820                 comma = 1;
2821         }
2822         if (t_flags & TF_SENTFIN) {
2823                 db_printf("%sTF_SENTFIN", comma ? ", " : "");
2824                 comma = 1;
2825         }
2826         if (t_flags & TF_REQ_SCALE) {
2827                 db_printf("%sTF_REQ_SCALE", comma ? ", " : "");
2828                 comma = 1;
2829         }
2830         if (t_flags & TF_RCVD_SCALE) {
2831                 db_printf("%sTF_RECVD_SCALE", comma ? ", " : "");
2832                 comma = 1;
2833         }
2834         if (t_flags & TF_REQ_TSTMP) {
2835                 db_printf("%sTF_REQ_TSTMP", comma ? ", " : "");
2836                 comma = 1;
2837         }
2838         if (t_flags & TF_RCVD_TSTMP) {
2839                 db_printf("%sTF_RCVD_TSTMP", comma ? ", " : "");
2840                 comma = 1;
2841         }
2842         if (t_flags & TF_SACK_PERMIT) {
2843                 db_printf("%sTF_SACK_PERMIT", comma ? ", " : "");
2844                 comma = 1;
2845         }
2846         if (t_flags & TF_NEEDSYN) {
2847                 db_printf("%sTF_NEEDSYN", comma ? ", " : "");
2848                 comma = 1;
2849         }
2850         if (t_flags & TF_NEEDFIN) {
2851                 db_printf("%sTF_NEEDFIN", comma ? ", " : "");
2852                 comma = 1;
2853         }
2854         if (t_flags & TF_NOPUSH) {
2855                 db_printf("%sTF_NOPUSH", comma ? ", " : "");
2856                 comma = 1;
2857         }
2858         if (t_flags & TF_PREVVALID) {
2859                 db_printf("%sTF_PREVVALID", comma ? ", " : "");
2860                 comma = 1;
2861         }
2862         if (t_flags & TF_MORETOCOME) {
2863                 db_printf("%sTF_MORETOCOME", comma ? ", " : "");
2864                 comma = 1;
2865         }
2866         if (t_flags & TF_SONOTCONN) {
2867                 db_printf("%sTF_SONOTCONN", comma ? ", " : "");
2868                 comma = 1;
2869         }
2870         if (t_flags & TF_LASTIDLE) {
2871                 db_printf("%sTF_LASTIDLE", comma ? ", " : "");
2872                 comma = 1;
2873         }
2874         if (t_flags & TF_RXWIN0SENT) {
2875                 db_printf("%sTF_RXWIN0SENT", comma ? ", " : "");
2876                 comma = 1;
2877         }
2878         if (t_flags & TF_FASTRECOVERY) {
2879                 db_printf("%sTF_FASTRECOVERY", comma ? ", " : "");
2880                 comma = 1;
2881         }
2882         if (t_flags & TF_CONGRECOVERY) {
2883                 db_printf("%sTF_CONGRECOVERY", comma ? ", " : "");
2884                 comma = 1;
2885         }
2886         if (t_flags & TF_WASFRECOVERY) {
2887                 db_printf("%sTF_WASFRECOVERY", comma ? ", " : "");
2888                 comma = 1;
2889         }
2890         if (t_flags & TF_WASCRECOVERY) {
2891                 db_printf("%sTF_WASCRECOVERY", comma ? ", " : "");
2892                 comma = 1;
2893         }
2894         if (t_flags & TF_SIGNATURE) {
2895                 db_printf("%sTF_SIGNATURE", comma ? ", " : "");
2896                 comma = 1;
2897         }
2898         if (t_flags & TF_FORCEDATA) {
2899                 db_printf("%sTF_FORCEDATA", comma ? ", " : "");
2900                 comma = 1;
2901         }
2902         if (t_flags & TF_TSO) {
2903                 db_printf("%sTF_TSO", comma ? ", " : "");
2904                 comma = 1;
2905         }
2906         if (t_flags & TF_FASTOPEN) {
2907                 db_printf("%sTF_FASTOPEN", comma ? ", " : "");
2908                 comma = 1;
2909         }
2910 }
2911
2912 static void
2913 db_print_tflags2(u_int t_flags2)
2914 {
2915         int comma;
2916
2917         comma = 0;
2918         if (t_flags2 & TF2_PLPMTU_BLACKHOLE) {
2919                 db_printf("%sTF2_PLPMTU_BLACKHOLE", comma ? ", " : "");
2920                 comma = 1;
2921         }
2922         if (t_flags2 & TF2_PLPMTU_PMTUD) {
2923                 db_printf("%sTF2_PLPMTU_PMTUD", comma ? ", " : "");
2924                 comma = 1;
2925         }
2926         if (t_flags2 & TF2_PLPMTU_MAXSEGSNT) {
2927                 db_printf("%sTF2_PLPMTU_MAXSEGSNT", comma ? ", " : "");
2928                 comma = 1;
2929         }
2930         if (t_flags2 & TF2_LOG_AUTO) {
2931                 db_printf("%sTF2_LOG_AUTO", comma ? ", " : "");
2932                 comma = 1;
2933         }
2934         if (t_flags2 & TF2_DROP_AF_DATA) {
2935                 db_printf("%sTF2_DROP_AF_DATA", comma ? ", " : "");
2936                 comma = 1;
2937         }
2938         if (t_flags2 & TF2_ECN_PERMIT) {
2939                 db_printf("%sTF2_ECN_PERMIT", comma ? ", " : "");
2940                 comma = 1;
2941         }
2942         if (t_flags2 & TF2_ECN_SND_CWR) {
2943                 db_printf("%sTF2_ECN_SND_CWR", comma ? ", " : "");
2944                 comma = 1;
2945         }
2946         if (t_flags2 & TF2_ECN_SND_ECE) {
2947                 db_printf("%sTF2_ECN_SND_ECE", comma ? ", " : "");
2948                 comma = 1;
2949         }
2950         if (t_flags2 & TF2_ACE_PERMIT) {
2951                 db_printf("%sTF2_ACE_PERMIT", comma ? ", " : "");
2952                 comma = 1;
2953         }
2954         if (t_flags2 & TF2_FBYTES_COMPLETE) {
2955                 db_printf("%sTF2_FBYTES_COMPLETE", comma ? ", " : "");
2956                 comma = 1;
2957         }
2958 }
2959
2960 static void
2961 db_print_toobflags(char t_oobflags)
2962 {
2963         int comma;
2964
2965         comma = 0;
2966         if (t_oobflags & TCPOOB_HAVEDATA) {
2967                 db_printf("%sTCPOOB_HAVEDATA", comma ? ", " : "");
2968                 comma = 1;
2969         }
2970         if (t_oobflags & TCPOOB_HADDATA) {
2971                 db_printf("%sTCPOOB_HADDATA", comma ? ", " : "");
2972                 comma = 1;
2973         }
2974 }
2975
2976 static void
2977 db_print_tcpcb(struct tcpcb *tp, const char *name, int indent)
2978 {
2979
2980         db_print_indent(indent);
2981         db_printf("%s at %p\n", name, tp);
2982
2983         indent += 2;
2984
2985         db_print_indent(indent);
2986         db_printf("t_segq first: %p   t_segqlen: %d   t_dupacks: %d\n",
2987            TAILQ_FIRST(&tp->t_segq), tp->t_segqlen, tp->t_dupacks);
2988
2989         db_print_indent(indent);
2990         db_printf("t_callout: %p   t_timers: %p\n",
2991             &tp->t_callout, &tp->t_timers);
2992
2993         db_print_indent(indent);
2994         db_printf("t_state: %d (", tp->t_state);
2995         db_print_tstate(tp->t_state);
2996         db_printf(")\n");
2997
2998         db_print_indent(indent);
2999         db_printf("t_flags: 0x%x (", tp->t_flags);
3000         db_print_tflags(tp->t_flags);
3001         db_printf(")\n");
3002
3003         db_print_indent(indent);
3004         db_printf("t_flags2: 0x%x (", tp->t_flags2);
3005         db_print_tflags2(tp->t_flags2);
3006         db_printf(")\n");
3007
3008         db_print_indent(indent);
3009         db_printf("snd_una: 0x%08x   snd_max: 0x%08x   snd_nxt: x0%08x\n",
3010             tp->snd_una, tp->snd_max, tp->snd_nxt);
3011
3012         db_print_indent(indent);
3013         db_printf("snd_up: 0x%08x   snd_wl1: 0x%08x   snd_wl2: 0x%08x\n",
3014            tp->snd_up, tp->snd_wl1, tp->snd_wl2);
3015
3016         db_print_indent(indent);
3017         db_printf("iss: 0x%08x   irs: 0x%08x   rcv_nxt: 0x%08x\n",
3018             tp->iss, tp->irs, tp->rcv_nxt);
3019
3020         db_print_indent(indent);
3021         db_printf("rcv_adv: 0x%08x   rcv_wnd: %u   rcv_up: 0x%08x\n",
3022             tp->rcv_adv, tp->rcv_wnd, tp->rcv_up);
3023
3024         db_print_indent(indent);
3025         db_printf("snd_wnd: %u   snd_cwnd: %u\n",
3026            tp->snd_wnd, tp->snd_cwnd);
3027
3028         db_print_indent(indent);
3029         db_printf("snd_ssthresh: %u   snd_recover: "
3030             "0x%08x\n", tp->snd_ssthresh, tp->snd_recover);
3031
3032         db_print_indent(indent);
3033         db_printf("t_rcvtime: %u   t_startime: %u\n",
3034             tp->t_rcvtime, tp->t_starttime);
3035
3036         db_print_indent(indent);
3037         db_printf("t_rttime: %u   t_rtsq: 0x%08x\n",
3038             tp->t_rtttime, tp->t_rtseq);
3039
3040         db_print_indent(indent);
3041         db_printf("t_rxtcur: %d   t_maxseg: %u   t_srtt: %d\n",
3042             tp->t_rxtcur, tp->t_maxseg, tp->t_srtt);
3043
3044         db_print_indent(indent);
3045         db_printf("t_rttvar: %d   t_rxtshift: %d   t_rttmin: %u\n",
3046             tp->t_rttvar, tp->t_rxtshift, tp->t_rttmin);
3047
3048         db_print_indent(indent);
3049         db_printf("t_rttupdated: %u   max_sndwnd: %u   t_softerror: %d\n",
3050             tp->t_rttupdated, tp->max_sndwnd, tp->t_softerror);
3051
3052         db_print_indent(indent);
3053         db_printf("t_oobflags: 0x%x (", tp->t_oobflags);
3054         db_print_toobflags(tp->t_oobflags);
3055         db_printf(")   t_iobc: 0x%02x\n", tp->t_iobc);
3056
3057         db_print_indent(indent);
3058         db_printf("snd_scale: %u   rcv_scale: %u   request_r_scale: %u\n",
3059             tp->snd_scale, tp->rcv_scale, tp->request_r_scale);
3060
3061         db_print_indent(indent);
3062         db_printf("ts_recent: %u   ts_recent_age: %u\n",
3063             tp->ts_recent, tp->ts_recent_age);
3064
3065         db_print_indent(indent);
3066         db_printf("ts_offset: %u   last_ack_sent: 0x%08x   snd_cwnd_prev: "
3067             "%u\n", tp->ts_offset, tp->last_ack_sent, tp->snd_cwnd_prev);
3068
3069         db_print_indent(indent);
3070         db_printf("snd_ssthresh_prev: %u   snd_recover_prev: 0x%08x   "
3071             "t_badrxtwin: %u\n", tp->snd_ssthresh_prev,
3072             tp->snd_recover_prev, tp->t_badrxtwin);
3073
3074         db_print_indent(indent);
3075         db_printf("snd_numholes: %d  snd_holes first: %p\n",
3076             tp->snd_numholes, TAILQ_FIRST(&tp->snd_holes));
3077
3078         db_print_indent(indent);
3079         db_printf("snd_fack: 0x%08x   rcv_numsacks: %d\n",
3080             tp->snd_fack, tp->rcv_numsacks);
3081
3082         /* Skip sackblks, sackhint. */
3083
3084         db_print_indent(indent);
3085         db_printf("t_rttlow: %d   rfbuf_ts: %u   rfbuf_cnt: %d\n",
3086             tp->t_rttlow, tp->rfbuf_ts, tp->rfbuf_cnt);
3087 }
3088
3089 DB_SHOW_COMMAND(tcpcb, db_show_tcpcb)
3090 {
3091         struct tcpcb *tp;
3092
3093         if (!have_addr) {
3094                 db_printf("usage: show tcpcb <addr>\n");
3095                 return;
3096         }
3097         tp = (struct tcpcb *)addr;
3098
3099         db_print_tcpcb(tp, "tcpcb", 0);
3100 }
3101 #endif