sys/netinet/tcp_subr.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. Neither the name of the University nor the names of its contributors
  16  *    may be used to endorse or promote products derived from this software
  17  *    without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  *      @(#)tcp_subr.c  8.2 (Berkeley) 5/24/95
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_inet.h"
  38 #include "opt_inet6.h"
  39 #include "opt_ipsec.h"
  40 #include "opt_kern_tls.h"
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/arb.h>
  45 #include <sys/callout.h>
  46 #include <sys/eventhandler.h>
  47 #ifdef TCP_HHOOK
  48 #include <sys/hhook.h>
  49 #endif
  50 #include <sys/kernel.h>
  51 #ifdef TCP_HHOOK
  52 #include <sys/khelp.h>
  53 #endif
  54 #ifdef KERN_TLS
  55 #include <sys/ktls.h>
  56 #endif
  57 #include <sys/qmath.h>
  58 #include <sys/stats.h>
  59 #include <sys/sysctl.h>
  60 #include <sys/jail.h>
  61 #include <sys/malloc.h>
  62 #include <sys/refcount.h>
  63 #include <sys/mbuf.h>
  64 #include <sys/priv.h>
  65 #include <sys/proc.h>
  66 #include <sys/sdt.h>
  67 #include <sys/socket.h>
  68 #include <sys/socketvar.h>
  69 #include <sys/protosw.h>
  70 #include <sys/random.h>
  71
  72 #include <vm/uma.h>
  73
  74 #include <net/route.h>
  75 #include <net/route/nhop.h>
  76 #include <net/if.h>
  77 #include <net/if_var.h>
  78 #include <net/if_private.h>
  79 #include <net/vnet.h>
  80
  81 #include <netinet/in.h>
  82 #include <netinet/in_fib.h>
  83 #include <netinet/in_kdtrace.h>
  84 #include <netinet/in_pcb.h>
  85 #include <netinet/in_systm.h>
  86 #include <netinet/in_var.h>
  87 #include <netinet/ip.h>
  88 #include <netinet/ip_icmp.h>
  89 #include <netinet/ip_var.h>
  90 #ifdef INET6
  91 #include <netinet/icmp6.h>
  92 #include <netinet/ip6.h>
  93 #include <netinet6/in6_fib.h>
  94 #include <netinet6/in6_pcb.h>
  95 #include <netinet6/ip6_var.h>
  96 #include <netinet6/scope6_var.h>
  97 #include <netinet6/nd6.h>
  98 #endif
  99
 100 #include <netinet/tcp.h>
 101 #ifdef INVARIANTS
 102 #define TCPSTATES
 103 #endif
 104 #include <netinet/tcp_fsm.h>
 105 #include <netinet/tcp_seq.h>
 106 #include <netinet/tcp_timer.h>
 107 #include <netinet/tcp_var.h>
 108 #include <netinet/tcp_ecn.h>
 109 #include <netinet/tcp_log_buf.h>
 110 #include <netinet/tcp_syncache.h>
 111 #include <netinet/tcp_hpts.h>
 112 #include <netinet/tcp_lro.h>
 113 #include <netinet/cc/cc.h>
 114 #include <netinet/tcpip.h>
 115 #include <netinet/tcp_fastopen.h>
 116 #include <netinet/tcp_accounting.h>
 117 #ifdef TCPPCAP
 118 #include <netinet/tcp_pcap.h>
 119 #endif
 120 #ifdef TCP_OFFLOAD
 121 #include <netinet/tcp_offload.h>
 122 #endif
 123 #include <netinet/udp.h>
 124 #include <netinet/udp_var.h>
 125 #ifdef INET6
 126 #include <netinet6/tcp6_var.h>
 127 #endif
 128
 129 #include <netipsec/ipsec_support.h>
 130
 131 #include <machine/in_cksum.h>
 132 #include <crypto/siphash/siphash.h>
 133
 134 #include <security/mac/mac_framework.h>
 135
 136 #ifdef INET6
 137 static ip6proto_ctlinput_t tcp6_ctlinput;
 138 static udp_tun_icmp_t tcp6_ctlinput_viaudp;
 139 #endif
 140
 141 VNET_DEFINE(int, tcp_mssdflt) = TCP_MSS;
 142 #ifdef INET6
 143 VNET_DEFINE(int, tcp_v6mssdflt) = TCP6_MSS;
 144 #endif
 145
 146 #ifdef TCP_SAD_DETECTION
 147 /*  Sack attack detection thresholds and such */
 148 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, sack_attack,
 149     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 150     "Sack Attack detection thresholds");
 151 int32_t tcp_force_detection = 0;
 152 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, force_detection,
 153     CTLFLAG_RW,
 154     &tcp_force_detection, 0,
 155     "Do we force detection even if the INP has it off?");
 156 int32_t tcp_sad_limit = 10000;
 157 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, limit,
 158     CTLFLAG_RW,
 159     &tcp_sad_limit, 10000,
 160     "If SaD is enabled, what is the limit to sendmap entries (0 = unlimited)?");
 161 int32_t tcp_sack_to_ack_thresh = 700;   /* 70 % */
 162 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sack_to_ack_thresh,
 163     CTLFLAG_RW,
 164     &tcp_sack_to_ack_thresh, 700,
 165     "Percentage of sacks to acks we must see above (10.1 percent is 101)?");
 166 int32_t tcp_sack_to_move_thresh = 600;  /* 60 % */
 167 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, move_thresh,
 168     CTLFLAG_RW,
 169     &tcp_sack_to_move_thresh, 600,
 170     "Percentage of sack moves we must see above (10.1 percent is 101)");
 171 int32_t tcp_restoral_thresh = 450;      /* 45 % (sack:2:ack -25%) (mv:ratio -15%) **/
 172 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, restore_thresh,
 173     CTLFLAG_RW,
 174     &tcp_restoral_thresh, 450,
 175     "Percentage of sack to ack percentage we must see below to restore(10.1 percent is 101)");
 176 int32_t tcp_sad_decay_val = 800;
 177 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, decay_per,
 178     CTLFLAG_RW,
 179     &tcp_sad_decay_val, 800,
 180     "The decay percentage (10.1 percent equals 101 )");
 181 int32_t tcp_map_minimum = 500;
 182 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, nummaps,
 183     CTLFLAG_RW,
 184     &tcp_map_minimum, 500,
 185     "Number of Map enteries before we start detection");
 186 int32_t tcp_sad_pacing_interval = 2000;
 187 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sad_pacing_int,
 188     CTLFLAG_RW,
 189     &tcp_sad_pacing_interval, 2000,
 190     "What is the minimum pacing interval for a classified attacker?");
 191
 192 int32_t tcp_sad_low_pps = 100;
 193 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sad_low_pps,
 194     CTLFLAG_RW,
 195     &tcp_sad_low_pps, 100,
 196     "What is the input pps that below which we do not decay?");
 197 #endif
 198 uint32_t tcp_ack_war_time_window = 1000;
 199 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ack_war_timewindow,
 200     CTLFLAG_RW,
 201     &tcp_ack_war_time_window, 1000,
 202    "If the tcp_stack does ack-war prevention how many milliseconds are in its time window?");
 203 uint32_t tcp_ack_war_cnt = 5;
 204 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ack_war_cnt,
 205     CTLFLAG_RW,
 206     &tcp_ack_war_cnt, 5,
 207    "If the tcp_stack does ack-war prevention how many acks can be sent in its time window?");
 208
 209 struct rwlock tcp_function_lock;
 210
 211 static int
 212 sysctl_net_inet_tcp_mss_check(SYSCTL_HANDLER_ARGS)
 213 {
 214         int error, new;
 215
 216         new = V_tcp_mssdflt;
 217         error = sysctl_handle_int(oidp, &new, 0, req);
 218         if (error == 0 && req->newptr) {
 219                 if (new < TCP_MINMSS)
 220                         error = EINVAL;
 221                 else
 222                         V_tcp_mssdflt = new;
 223         }
 224         return (error);
 225 }
 226
 227 SYSCTL_PROC(_net_inet_tcp, TCPCTL_MSSDFLT, mssdflt,
 228     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 229     &VNET_NAME(tcp_mssdflt), 0, &sysctl_net_inet_tcp_mss_check, "I",
 230     "Default TCP Maximum Segment Size");
 231
 232 #ifdef INET6
 233 static int
 234 sysctl_net_inet_tcp_mss_v6_check(SYSCTL_HANDLER_ARGS)
 235 {
 236         int error, new;
 237
 238         new = V_tcp_v6mssdflt;
 239         error = sysctl_handle_int(oidp, &new, 0, req);
 240         if (error == 0 && req->newptr) {
 241                 if (new < TCP_MINMSS)
 242                         error = EINVAL;
 243                 else
 244                         V_tcp_v6mssdflt = new;
 245         }
 246         return (error);
 247 }
 248
 249 SYSCTL_PROC(_net_inet_tcp, TCPCTL_V6MSSDFLT, v6mssdflt,
 250     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 251     &VNET_NAME(tcp_v6mssdflt), 0, &sysctl_net_inet_tcp_mss_v6_check, "I",
 252    "Default TCP Maximum Segment Size for IPv6");
 253 #endif /* INET6 */
 254
 255 /*
 256  * Minimum MSS we accept and use. This prevents DoS attacks where
 257  * we are forced to a ridiculous low MSS like 20 and send hundreds
 258  * of packets instead of one. The effect scales with the available
 259  * bandwidth and quickly saturates the CPU and network interface
 260  * with packet generation and sending. Set to zero to disable MINMSS
 261  * checking. This setting prevents us from sending too small packets.
 262  */
 263 VNET_DEFINE(int, tcp_minmss) = TCP_MINMSS;
 264 SYSCTL_INT(_net_inet_tcp, OID_AUTO, minmss, CTLFLAG_VNET | CTLFLAG_RW,
 265      &VNET_NAME(tcp_minmss), 0,
 266     "Minimum TCP Maximum Segment Size");
 267
 268 VNET_DEFINE(int, tcp_do_rfc1323) = 1;
 269 SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1323, rfc1323, CTLFLAG_VNET | CTLFLAG_RW,
 270     &VNET_NAME(tcp_do_rfc1323), 0,
 271     "Enable rfc1323 (high performance TCP) extensions");
 272
 273 /*
 274  * As of June 2021, several TCP stacks violate RFC 7323 from September 2014.
 275  * Some stacks negotiate TS, but never send them after connection setup. Some
 276  * stacks negotiate TS, but don't send them when sending keep-alive segments.
 277  * These include modern widely deployed TCP stacks.
 278  * Therefore tolerating violations for now...
 279  */
 280 VNET_DEFINE(int, tcp_tolerate_missing_ts) = 1;
 281 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tolerate_missing_ts, CTLFLAG_VNET | CTLFLAG_RW,
 282     &VNET_NAME(tcp_tolerate_missing_ts), 0,
 283     "Tolerate missing TCP timestamps");
 284
 285 VNET_DEFINE(int, tcp_ts_offset_per_conn) = 1;
 286 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ts_offset_per_conn, CTLFLAG_VNET | CTLFLAG_RW,
 287     &VNET_NAME(tcp_ts_offset_per_conn), 0,
 288     "Initialize TCP timestamps per connection instead of per host pair");
 289
 290 /* How many connections are pacing */
 291 static volatile uint32_t number_of_tcp_connections_pacing = 0;
 292 static uint32_t shadow_num_connections = 0;
 293 static counter_u64_t tcp_pacing_failures;
 294
 295 static int tcp_pacing_limit = 10000;
 296 SYSCTL_INT(_net_inet_tcp, OID_AUTO, pacing_limit, CTLFLAG_RW,
 297     &tcp_pacing_limit, 1000,
 298     "If the TCP stack does pacing, is there a limit (-1 = no, 0 = no pacing N = number of connections)");
 299
 300 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pacing_count, CTLFLAG_RD,
 301     &shadow_num_connections, 0, "Number of TCP connections being paced");
 302
 303 SYSCTL_COUNTER_U64(_net_inet_tcp, OID_AUTO, pacing_failures, CTLFLAG_RD,
 304     &tcp_pacing_failures, "Number of times we failed to enable pacing to avoid exceeding the limit");
 305
 306 static int      tcp_log_debug = 0;
 307 SYSCTL_INT(_net_inet_tcp, OID_AUTO, log_debug, CTLFLAG_RW,
 308     &tcp_log_debug, 0, "Log errors caused by incoming TCP segments");
 309
 310 static int      tcp_tcbhashsize;
 311 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcbhashsize, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
 312     &tcp_tcbhashsize, 0, "Size of TCP control-block hashtable");
 313
 314 static int      do_tcpdrain = 1;
 315 SYSCTL_INT(_net_inet_tcp, OID_AUTO, do_tcpdrain, CTLFLAG_RW, &do_tcpdrain, 0,
 316     "Enable tcp_drain routine for extra help when low on mbufs");
 317
 318 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_VNET | CTLFLAG_RD,
 319     &VNET_NAME(tcbinfo.ipi_count), 0, "Number of active PCBs");
 320
 321 VNET_DEFINE_STATIC(int, icmp_may_rst) = 1;
 322 #define V_icmp_may_rst                  VNET(icmp_may_rst)
 323 SYSCTL_INT(_net_inet_tcp, OID_AUTO, icmp_may_rst, CTLFLAG_VNET | CTLFLAG_RW,
 324     &VNET_NAME(icmp_may_rst), 0,
 325     "Certain ICMP unreachable messages may abort connections in SYN_SENT");
 326
 327 VNET_DEFINE_STATIC(int, tcp_isn_reseed_interval) = 0;
 328 #define V_tcp_isn_reseed_interval       VNET(tcp_isn_reseed_interval)
 329 SYSCTL_INT(_net_inet_tcp, OID_AUTO, isn_reseed_interval, CTLFLAG_VNET | CTLFLAG_RW,
 330     &VNET_NAME(tcp_isn_reseed_interval), 0,
 331     "Seconds between reseeding of ISN secret");
 332
 333 static int      tcp_soreceive_stream;
 334 SYSCTL_INT(_net_inet_tcp, OID_AUTO, soreceive_stream, CTLFLAG_RDTUN,
 335     &tcp_soreceive_stream, 0, "Using soreceive_stream for TCP sockets");
 336
 337 VNET_DEFINE(uma_zone_t, sack_hole_zone);
 338 #define V_sack_hole_zone                VNET(sack_hole_zone)
 339 VNET_DEFINE(uint32_t, tcp_map_entries_limit) = 0;       /* unlimited */
 340 static int
 341 sysctl_net_inet_tcp_map_limit_check(SYSCTL_HANDLER_ARGS)
 342 {
 343         int error;
 344         uint32_t new;
 345
 346         new = V_tcp_map_entries_limit;
 347         error = sysctl_handle_int(oidp, &new, 0, req);
 348         if (error == 0 && req->newptr) {
 349                 /* only allow "0" and value > minimum */
 350                 if (new > 0 && new < TCP_MIN_MAP_ENTRIES_LIMIT)
 351                         error = EINVAL;
 352                 else
 353                         V_tcp_map_entries_limit = new;
 354         }
 355         return (error);
 356 }
 357 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, map_limit,
 358     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 359     &VNET_NAME(tcp_map_entries_limit), 0,
 360     &sysctl_net_inet_tcp_map_limit_check, "IU",
 361     "Total sendmap entries limit");
 362
 363 VNET_DEFINE(uint32_t, tcp_map_split_limit) = 0; /* unlimited */
 364 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, split_limit, CTLFLAG_VNET | CTLFLAG_RW,
 365      &VNET_NAME(tcp_map_split_limit), 0,
 366     "Total sendmap split entries limit");
 367
 368 #ifdef TCP_HHOOK
 369 VNET_DEFINE(struct hhook_head *, tcp_hhh[HHOOK_TCP_LAST+1]);
 370 #endif
 371
 372 #define TS_OFFSET_SECRET_LENGTH SIPHASH_KEY_LENGTH
 373 VNET_DEFINE_STATIC(u_char, ts_offset_secret[TS_OFFSET_SECRET_LENGTH]);
 374 #define V_ts_offset_secret      VNET(ts_offset_secret)
 375
 376 static int      tcp_default_fb_init(struct tcpcb *tp, void **ptr);
 377 static void     tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged);
 378 static int      tcp_default_handoff_ok(struct tcpcb *tp);
 379 static struct inpcb *tcp_notify(struct inpcb *, int);
 380 static struct inpcb *tcp_mtudisc_notify(struct inpcb *, int);
 381 static struct inpcb *tcp_mtudisc(struct inpcb *, int);
 382 static struct inpcb *tcp_drop_syn_sent(struct inpcb *, int);
 383 static char *   tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th,
 384                     const void *ip4hdr, const void *ip6hdr);
 385 static void     tcp_default_switch_failed(struct tcpcb *tp);
 386 static ipproto_ctlinput_t       tcp_ctlinput;
 387 static udp_tun_icmp_t           tcp_ctlinput_viaudp;
 388
 389 static struct tcp_function_block tcp_def_funcblk = {
 390         .tfb_tcp_block_name = "freebsd",
 391         .tfb_tcp_output = tcp_default_output,
 392         .tfb_tcp_do_segment = tcp_do_segment,
 393         .tfb_tcp_ctloutput = tcp_default_ctloutput,
 394         .tfb_tcp_handoff_ok = tcp_default_handoff_ok,
 395         .tfb_tcp_fb_init = tcp_default_fb_init,
 396         .tfb_tcp_fb_fini = tcp_default_fb_fini,
 397         .tfb_switch_failed = tcp_default_switch_failed,
 398 };
 399
 400 static int tcp_fb_cnt = 0;
 401 struct tcp_funchead t_functions;
 402 VNET_DEFINE_STATIC(struct tcp_function_block *, tcp_func_set_ptr) = &tcp_def_funcblk;
 403 #define V_tcp_func_set_ptr VNET(tcp_func_set_ptr)
 404
 405 void
 406 tcp_record_dsack(struct tcpcb *tp, tcp_seq start, tcp_seq end, int tlp)
 407 {
 408         TCPSTAT_INC(tcps_dsack_count);
 409         tp->t_dsack_pack++;
 410         if (tlp == 0) {
 411                 if (SEQ_GT(end, start)) {
 412                         tp->t_dsack_bytes += (end - start);
 413                         TCPSTAT_ADD(tcps_dsack_bytes, (end - start));
 414                 } else {
 415                         tp->t_dsack_tlp_bytes += (start - end);
 416                         TCPSTAT_ADD(tcps_dsack_bytes, (start - end));
 417                 }
 418         } else {
 419                 if (SEQ_GT(end, start)) {
 420                         tp->t_dsack_bytes += (end - start);
 421                         TCPSTAT_ADD(tcps_dsack_tlp_bytes, (end - start));
 422                 } else {
 423                         tp->t_dsack_tlp_bytes += (start - end);
 424                         TCPSTAT_ADD(tcps_dsack_tlp_bytes, (start - end));
 425                 }
 426         }
 427 }
 428
 429 static struct tcp_function_block *
 430 find_tcp_functions_locked(struct tcp_function_set *fs)
 431 {
 432         struct tcp_function *f;
 433         struct tcp_function_block *blk=NULL;
 434
 435         TAILQ_FOREACH(f, &t_functions, tf_next) {
 436                 if (strcmp(f->tf_name, fs->function_set_name) == 0) {
 437                         blk = f->tf_fb;
 438                         break;
 439                 }
 440         }
 441         return(blk);
 442 }
 443
 444 static struct tcp_function_block *
 445 find_tcp_fb_locked(struct tcp_function_block *blk, struct tcp_function **s)
 446 {
 447         struct tcp_function_block *rblk=NULL;
 448         struct tcp_function *f;
 449
 450         TAILQ_FOREACH(f, &t_functions, tf_next) {
 451                 if (f->tf_fb == blk) {
 452                         rblk = blk;
 453                         if (s) {
 454                                 *s = f;
 455                         }
 456                         break;
 457                 }
 458         }
 459         return (rblk);
 460 }
 461
 462 struct tcp_function_block *
 463 find_and_ref_tcp_functions(struct tcp_function_set *fs)
 464 {
 465         struct tcp_function_block *blk;
 466
 467         rw_rlock(&tcp_function_lock);
 468         blk = find_tcp_functions_locked(fs);
 469         if (blk)
 470                 refcount_acquire(&blk->tfb_refcnt);
 471         rw_runlock(&tcp_function_lock);
 472         return(blk);
 473 }
 474
 475 struct tcp_function_block *
 476 find_and_ref_tcp_fb(struct tcp_function_block *blk)
 477 {
 478         struct tcp_function_block *rblk;
 479
 480         rw_rlock(&tcp_function_lock);
 481         rblk = find_tcp_fb_locked(blk, NULL);
 482         if (rblk)
 483                 refcount_acquire(&rblk->tfb_refcnt);
 484         rw_runlock(&tcp_function_lock);
 485         return(rblk);
 486 }
 487
 488 /* Find a matching alias for the given tcp_function_block. */
 489 int
 490 find_tcp_function_alias(struct tcp_function_block *blk,
 491     struct tcp_function_set *fs)
 492 {
 493         struct tcp_function *f;
 494         int found;
 495
 496         found = 0;
 497         rw_rlock(&tcp_function_lock);
 498         TAILQ_FOREACH(f, &t_functions, tf_next) {
 499                 if ((f->tf_fb == blk) &&
 500                     (strncmp(f->tf_name, blk->tfb_tcp_block_name,
 501                         TCP_FUNCTION_NAME_LEN_MAX) != 0)) {
 502                         /* Matching function block with different name. */
 503                         strncpy(fs->function_set_name, f->tf_name,
 504                             TCP_FUNCTION_NAME_LEN_MAX);
 505                         found = 1;
 506                         break;
 507                 }
 508         }
 509         /* Null terminate the string appropriately. */
 510         if (found) {
 511                 fs->function_set_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0';
 512         } else {
 513                 fs->function_set_name[0] = '\0';
 514         }
 515         rw_runlock(&tcp_function_lock);
 516         return (found);
 517 }
 518
 519 static struct tcp_function_block *
 520 find_and_ref_tcp_default_fb(void)
 521 {
 522         struct tcp_function_block *rblk;
 523
 524         rw_rlock(&tcp_function_lock);
 525         rblk = V_tcp_func_set_ptr;
 526         refcount_acquire(&rblk->tfb_refcnt);
 527         rw_runlock(&tcp_function_lock);
 528         return (rblk);
 529 }
 530
 531 void
 532 tcp_switch_back_to_default(struct tcpcb *tp)
 533 {
 534         struct tcp_function_block *tfb;
 535         void *ptr = NULL;
 536
 537         KASSERT(tp->t_fb != &tcp_def_funcblk,
 538             ("%s: called by the built-in default stack", __func__));
 539
 540         /*
 541          * Now, we'll find a new function block to use.
 542          * Start by trying the current user-selected
 543          * default, unless this stack is the user-selected
 544          * default.
 545          */
 546         tfb = find_and_ref_tcp_default_fb();
 547         if (tfb == tp->t_fb) {
 548                 refcount_release(&tfb->tfb_refcnt);
 549                 tfb = NULL;
 550         }
 551         /* Does the stack accept this connection? */
 552         if (tfb != NULL && tfb->tfb_tcp_handoff_ok != NULL &&
 553             (*tfb->tfb_tcp_handoff_ok)(tp)) {
 554                 refcount_release(&tfb->tfb_refcnt);
 555                 tfb = NULL;
 556         }
 557         /* Try to use that stack. */
 558         if (tfb != NULL) {
 559                 /* Initialize the new stack. If it succeeds, we are done. */
 560                 if (tfb->tfb_tcp_fb_init == NULL ||
 561                     (*tfb->tfb_tcp_fb_init)(tp, &ptr) == 0) {
 562                         /* Release the old stack */
 563                         if (tp->t_fb->tfb_tcp_fb_fini != NULL)
 564                                 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
 565                         refcount_release(&tp->t_fb->tfb_refcnt);
 566                         /* Now set in all the pointers */
 567                         tp->t_fb = tfb;
 568                         tp->t_fb_ptr = ptr;
 569                         return;
 570                 }
 571                 /*
 572                  * Initialization failed. Release the reference count on
 573                  * the looked up default stack.
 574                  */
 575                 refcount_release(&tfb->tfb_refcnt);
 576         }
 577
 578         /*
 579          * If that wasn't feasible, use the built-in default
 580          * stack which is not allowed to reject anyone.
 581          */
 582         tfb = find_and_ref_tcp_fb(&tcp_def_funcblk);
 583         if (tfb == NULL) {
 584                 /* there always should be a default */
 585                 panic("Can't refer to tcp_def_funcblk");
 586         }
 587         if (tfb->tfb_tcp_handoff_ok != NULL) {
 588                 if ((*tfb->tfb_tcp_handoff_ok) (tp)) {
 589                         /* The default stack cannot say no */
 590                         panic("Default stack rejects a new session?");
 591                 }
 592         }
 593         if (tfb->tfb_tcp_fb_init != NULL &&
 594             (*tfb->tfb_tcp_fb_init)(tp, &ptr)) {
 595                 /* The default stack cannot fail */
 596                 panic("Default stack initialization failed");
 597         }
 598         /* Now release the old stack */
 599         if (tp->t_fb->tfb_tcp_fb_fini != NULL)
 600                 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
 601         refcount_release(&tp->t_fb->tfb_refcnt);
 602         /* And set in the pointers to the new */
 603         tp->t_fb = tfb;
 604         tp->t_fb_ptr = ptr;
 605 }
 606
 607 static bool
 608 tcp_recv_udp_tunneled_packet(struct mbuf *m, int off, struct inpcb *inp,
 609     const struct sockaddr *sa, void *ctx)
 610 {
 611         struct ip *iph;
 612 #ifdef INET6
 613         struct ip6_hdr *ip6;
 614 #endif
 615         struct udphdr *uh;
 616         struct tcphdr *th;
 617         int thlen;
 618         uint16_t port;
 619
 620         TCPSTAT_INC(tcps_tunneled_pkts);
 621         if ((m->m_flags & M_PKTHDR) == 0) {
 622                 /* Can't handle one that is not a pkt hdr */
 623                 TCPSTAT_INC(tcps_tunneled_errs);
 624                 goto out;
 625         }
 626         thlen = sizeof(struct tcphdr);
 627         if (m->m_len < off + sizeof(struct udphdr) + thlen &&
 628             (m =  m_pullup(m, off + sizeof(struct udphdr) + thlen)) == NULL) {
 629                 TCPSTAT_INC(tcps_tunneled_errs);
 630                 goto out;
 631         }
 632         iph = mtod(m, struct ip *);
 633         uh = (struct udphdr *)((caddr_t)iph + off);
 634         th = (struct tcphdr *)(uh + 1);
 635         thlen = th->th_off << 2;
 636         if (m->m_len < off + sizeof(struct udphdr) + thlen) {
 637                 m =  m_pullup(m, off + sizeof(struct udphdr) + thlen);
 638                 if (m == NULL) {
 639                         TCPSTAT_INC(tcps_tunneled_errs);
 640                         goto out;
 641                 } else {
 642                         iph = mtod(m, struct ip *);
 643                         uh = (struct udphdr *)((caddr_t)iph + off);
 644                         th = (struct tcphdr *)(uh + 1);
 645                 }
 646         }
 647         m->m_pkthdr.tcp_tun_port = port = uh->uh_sport;
 648         bcopy(th, uh, m->m_len - off);
 649         m->m_len -= sizeof(struct udphdr);
 650         m->m_pkthdr.len -= sizeof(struct udphdr);
 651         /*
 652          * We use the same algorithm for
 653          * both UDP and TCP for c-sum. So
 654          * the code in tcp_input will skip
 655          * the checksum. So we do nothing
 656          * with the flag (m->m_pkthdr.csum_flags).
 657          */
 658         switch (iph->ip_v) {
 659 #ifdef INET
 660         case IPVERSION:
 661                 iph->ip_len = htons(ntohs(iph->ip_len) - sizeof(struct udphdr));
 662                 tcp_input_with_port(&m, &off, IPPROTO_TCP, port);
 663                 break;
 664 #endif
 665 #ifdef INET6
 666         case IPV6_VERSION >> 4:
 667                 ip6 = mtod(m, struct ip6_hdr *);
 668                 ip6->ip6_plen = htons(ntohs(ip6->ip6_plen) - sizeof(struct udphdr));
 669                 tcp6_input_with_port(&m, &off, IPPROTO_TCP, port);
 670                 break;
 671 #endif
 672         default:
 673                 goto out;
 674                 break;
 675         }
 676         return (true);
 677 out:
 678         m_freem(m);
 679
 680         return (true);
 681 }
 682
 683 static int
 684 sysctl_net_inet_default_tcp_functions(SYSCTL_HANDLER_ARGS)
 685 {
 686         int error=ENOENT;
 687         struct tcp_function_set fs;
 688         struct tcp_function_block *blk;
 689
 690         memset(&fs, 0, sizeof(fs));
 691         rw_rlock(&tcp_function_lock);
 692         blk = find_tcp_fb_locked(V_tcp_func_set_ptr, NULL);
 693         if (blk) {
 694                 /* Found him */
 695                 strcpy(fs.function_set_name, blk->tfb_tcp_block_name);
 696                 fs.pcbcnt = blk->tfb_refcnt;
 697         }
 698         rw_runlock(&tcp_function_lock);
 699         error = sysctl_handle_string(oidp, fs.function_set_name,
 700                                      sizeof(fs.function_set_name), req);
 701
 702         /* Check for error or no change */
 703         if (error != 0 || req->newptr == NULL)
 704                 return(error);
 705
 706         rw_wlock(&tcp_function_lock);
 707         blk = find_tcp_functions_locked(&fs);
 708         if ((blk == NULL) ||
 709             (blk->tfb_flags & TCP_FUNC_BEING_REMOVED)) {
 710                 error = ENOENT;
 711                 goto done;
 712         }
 713         V_tcp_func_set_ptr = blk;
 714 done:
 715         rw_wunlock(&tcp_function_lock);
 716         return (error);
 717 }
 718
 719 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_default,
 720     CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 721     NULL, 0, sysctl_net_inet_default_tcp_functions, "A",
 722     "Set/get the default TCP functions");
 723
 724 static int
 725 sysctl_net_inet_list_available(SYSCTL_HANDLER_ARGS)
 726 {
 727         int error, cnt, linesz;
 728         struct tcp_function *f;
 729         char *buffer, *cp;
 730         size_t bufsz, outsz;
 731         bool alias;
 732
 733         cnt = 0;
 734         rw_rlock(&tcp_function_lock);
 735         TAILQ_FOREACH(f, &t_functions, tf_next) {
 736                 cnt++;
 737         }
 738         rw_runlock(&tcp_function_lock);
 739
 740         bufsz = (cnt+2) * ((TCP_FUNCTION_NAME_LEN_MAX * 2) + 13) + 1;
 741         buffer = malloc(bufsz, M_TEMP, M_WAITOK);
 742
 743         error = 0;
 744         cp = buffer;
 745
 746         linesz = snprintf(cp, bufsz, "\n%-32s%c %-32s %s\n", "Stack", 'D',
 747             "Alias", "PCB count");
 748         cp += linesz;
 749         bufsz -= linesz;
 750         outsz = linesz;
 751
 752         rw_rlock(&tcp_function_lock);
 753         TAILQ_FOREACH(f, &t_functions, tf_next) {
 754                 alias = (f->tf_name != f->tf_fb->tfb_tcp_block_name);
 755                 linesz = snprintf(cp, bufsz, "%-32s%c %-32s %u\n",
 756                     f->tf_fb->tfb_tcp_block_name,
 757                     (f->tf_fb == V_tcp_func_set_ptr) ? '*' : ' ',
 758                     alias ? f->tf_name : "-",
 759                     f->tf_fb->tfb_refcnt);
 760                 if (linesz >= bufsz) {
 761                         error = EOVERFLOW;
 762                         break;
 763                 }
 764                 cp += linesz;
 765                 bufsz -= linesz;
 766                 outsz += linesz;
 767         }
 768         rw_runlock(&tcp_function_lock);
 769         if (error == 0)
 770                 error = sysctl_handle_string(oidp, buffer, outsz + 1, req);
 771         free(buffer, M_TEMP);
 772         return (error);
 773 }
 774
 775 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_available,
 776     CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
 777     NULL, 0, sysctl_net_inet_list_available, "A",
 778     "list available TCP Function sets");
 779
 780 VNET_DEFINE(int, tcp_udp_tunneling_port) = TCP_TUNNELING_PORT_DEFAULT;
 781
 782 #ifdef INET
 783 VNET_DEFINE(struct socket *, udp4_tun_socket) = NULL;
 784 #define V_udp4_tun_socket       VNET(udp4_tun_socket)
 785 #endif
 786 #ifdef INET6
 787 VNET_DEFINE(struct socket *, udp6_tun_socket) = NULL;
 788 #define V_udp6_tun_socket       VNET(udp6_tun_socket)
 789 #endif
 790
 791 static struct sx tcpoudp_lock;
 792
 793 static void
 794 tcp_over_udp_stop(void)
 795 {
 796
 797         sx_assert(&tcpoudp_lock, SA_XLOCKED);
 798
 799 #ifdef INET
 800         if (V_udp4_tun_socket != NULL) {
 801                 soclose(V_udp4_tun_socket);
 802                 V_udp4_tun_socket = NULL;
 803         }
 804 #endif
 805 #ifdef INET6
 806         if (V_udp6_tun_socket != NULL) {
 807                 soclose(V_udp6_tun_socket);
 808                 V_udp6_tun_socket = NULL;
 809         }
 810 #endif
 811 }
 812
 813 static int
 814 tcp_over_udp_start(void)
 815 {
 816         uint16_t port;
 817         int ret;
 818 #ifdef INET
 819         struct sockaddr_in sin;
 820 #endif
 821 #ifdef INET6
 822         struct sockaddr_in6 sin6;
 823 #endif
 824
 825         sx_assert(&tcpoudp_lock, SA_XLOCKED);
 826
 827         port = V_tcp_udp_tunneling_port;
 828         if (ntohs(port) == 0) {
 829                 /* Must have a port set */
 830                 return (EINVAL);
 831         }
 832 #ifdef INET
 833         if (V_udp4_tun_socket != NULL) {
 834                 /* Already running -- must stop first */
 835                 return (EALREADY);
 836         }
 837 #endif
 838 #ifdef INET6
 839         if (V_udp6_tun_socket != NULL) {
 840                 /* Already running -- must stop first */
 841                 return (EALREADY);
 842         }
 843 #endif
 844 #ifdef INET
 845         if ((ret = socreate(PF_INET, &V_udp4_tun_socket,
 846             SOCK_DGRAM, IPPROTO_UDP,
 847             curthread->td_ucred, curthread))) {
 848                 tcp_over_udp_stop();
 849                 return (ret);
 850         }
 851         /* Call the special UDP hook. */
 852         if ((ret = udp_set_kernel_tunneling(V_udp4_tun_socket,
 853             tcp_recv_udp_tunneled_packet,
 854             tcp_ctlinput_viaudp,
 855             NULL))) {
 856                 tcp_over_udp_stop();
 857                 return (ret);
 858         }
 859         /* Ok, we have a socket, bind it to the port. */
 860         memset(&sin, 0, sizeof(struct sockaddr_in));
 861         sin.sin_len = sizeof(struct sockaddr_in);
 862         sin.sin_family = AF_INET;
 863         sin.sin_port = htons(port);
 864         if ((ret = sobind(V_udp4_tun_socket,
 865             (struct sockaddr *)&sin, curthread))) {
 866                 tcp_over_udp_stop();
 867                 return (ret);
 868         }
 869 #endif
 870 #ifdef INET6
 871         if ((ret = socreate(PF_INET6, &V_udp6_tun_socket,
 872             SOCK_DGRAM, IPPROTO_UDP,
 873             curthread->td_ucred, curthread))) {
 874                 tcp_over_udp_stop();
 875                 return (ret);
 876         }
 877         /* Call the special UDP hook. */
 878         if ((ret = udp_set_kernel_tunneling(V_udp6_tun_socket,
 879             tcp_recv_udp_tunneled_packet,
 880             tcp6_ctlinput_viaudp,
 881             NULL))) {
 882                 tcp_over_udp_stop();
 883                 return (ret);
 884         }
 885         /* Ok, we have a socket, bind it to the port. */
 886         memset(&sin6, 0, sizeof(struct sockaddr_in6));
 887         sin6.sin6_len = sizeof(struct sockaddr_in6);
 888         sin6.sin6_family = AF_INET6;
 889         sin6.sin6_port = htons(port);
 890         if ((ret = sobind(V_udp6_tun_socket,
 891             (struct sockaddr *)&sin6, curthread))) {
 892                 tcp_over_udp_stop();
 893                 return (ret);
 894         }
 895 #endif
 896         return (0);
 897 }
 898
 899 static int
 900 sysctl_net_inet_tcp_udp_tunneling_port_check(SYSCTL_HANDLER_ARGS)
 901 {
 902         int error;
 903         uint32_t old, new;
 904
 905         old = V_tcp_udp_tunneling_port;
 906         new = old;
 907         error = sysctl_handle_int(oidp, &new, 0, req);
 908         if ((error == 0) &&
 909             (req->newptr != NULL)) {
 910                 if ((new < TCP_TUNNELING_PORT_MIN) ||
 911                     (new > TCP_TUNNELING_PORT_MAX)) {
 912                         error = EINVAL;
 913                 } else {
 914                         sx_xlock(&tcpoudp_lock);
 915                         V_tcp_udp_tunneling_port = new;
 916                         if (old != 0) {
 917                                 tcp_over_udp_stop();
 918                         }
 919                         if (new != 0) {
 920                                 error = tcp_over_udp_start();
 921                                 if (error != 0) {
 922                                         V_tcp_udp_tunneling_port = 0;
 923                                 }
 924                         }
 925                         sx_xunlock(&tcpoudp_lock);
 926                 }
 927         }
 928         return (error);
 929 }
 930
 931 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, udp_tunneling_port,
 932     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 933     &VNET_NAME(tcp_udp_tunneling_port),
 934     0, &sysctl_net_inet_tcp_udp_tunneling_port_check, "IU",
 935     "Tunneling port for tcp over udp");
 936
 937 VNET_DEFINE(int, tcp_udp_tunneling_overhead) = TCP_TUNNELING_OVERHEAD_DEFAULT;
 938
 939 static int
 940 sysctl_net_inet_tcp_udp_tunneling_overhead_check(SYSCTL_HANDLER_ARGS)
 941 {
 942         int error, new;
 943
 944         new = V_tcp_udp_tunneling_overhead;
 945         error = sysctl_handle_int(oidp, &new, 0, req);
 946         if (error == 0 && req->newptr) {
 947                 if ((new < TCP_TUNNELING_OVERHEAD_MIN) ||
 948                     (new > TCP_TUNNELING_OVERHEAD_MAX))
 949                         error = EINVAL;
 950                 else
 951                         V_tcp_udp_tunneling_overhead = new;
 952         }
 953         return (error);
 954 }
 955
 956 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, udp_tunneling_overhead,
 957     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 958     &VNET_NAME(tcp_udp_tunneling_overhead),
 959     0, &sysctl_net_inet_tcp_udp_tunneling_overhead_check, "IU",
 960     "MSS reduction when using tcp over udp");
 961
 962 /*
 963  * Exports one (struct tcp_function_info) for each alias/name.
 964  */
 965 static int
 966 sysctl_net_inet_list_func_info(SYSCTL_HANDLER_ARGS)
 967 {
 968         int cnt, error;
 969         struct tcp_function *f;
 970         struct tcp_function_info tfi;
 971
 972         /*
 973          * We don't allow writes.
 974          */
 975         if (req->newptr != NULL)
 976                 return (EINVAL);
 977
 978         /*
 979          * Wire the old buffer so we can directly copy the functions to
 980          * user space without dropping the lock.
 981          */
 982         if (req->oldptr != NULL) {
 983                 error = sysctl_wire_old_buffer(req, 0);
 984                 if (error)
 985                         return (error);
 986         }
 987
 988         /*
 989          * Walk the list and copy out matching entries. If INVARIANTS
 990          * is compiled in, also walk the list to verify the length of
 991          * the list matches what we have recorded.
 992          */
 993         rw_rlock(&tcp_function_lock);
 994
 995         cnt = 0;
 996 #ifndef INVARIANTS
 997         if (req->oldptr == NULL) {
 998                 cnt = tcp_fb_cnt;
 999                 goto skip_loop;
1000         }
1001 #endif
1002         TAILQ_FOREACH(f, &t_functions, tf_next) {
1003 #ifdef INVARIANTS
1004                 cnt++;
1005 #endif
1006                 if (req->oldptr != NULL) {
1007                         bzero(&tfi, sizeof(tfi));
1008                         tfi.tfi_refcnt = f->tf_fb->tfb_refcnt;
1009                         tfi.tfi_id = f->tf_fb->tfb_id;
1010                         (void)strlcpy(tfi.tfi_alias, f->tf_name,
1011                             sizeof(tfi.tfi_alias));
1012                         (void)strlcpy(tfi.tfi_name,
1013                             f->tf_fb->tfb_tcp_block_name, sizeof(tfi.tfi_name));
1014                         error = SYSCTL_OUT(req, &tfi, sizeof(tfi));
1015                         /*
1016                          * Don't stop on error, as that is the
1017                          * mechanism we use to accumulate length
1018                          * information if the buffer was too short.
1019                          */
1020                 }
1021         }
1022         KASSERT(cnt == tcp_fb_cnt,
1023             ("%s: cnt (%d) != tcp_fb_cnt (%d)", __func__, cnt, tcp_fb_cnt));
1024 #ifndef INVARIANTS
1025 skip_loop:
1026 #endif
1027         rw_runlock(&tcp_function_lock);
1028         if (req->oldptr == NULL)
1029                 error = SYSCTL_OUT(req, NULL,
1030                     (cnt + 1) * sizeof(struct tcp_function_info));
1031
1032         return (error);
1033 }
1034
1035 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, function_info,
1036             CTLTYPE_OPAQUE | CTLFLAG_SKIP | CTLFLAG_RD | CTLFLAG_MPSAFE,
1037             NULL, 0, sysctl_net_inet_list_func_info, "S,tcp_function_info",
1038             "List TCP function block name-to-ID mappings");
1039
1040 /*
1041  * tfb_tcp_handoff_ok() function for the default stack.
1042  * Note that we'll basically try to take all comers.
1043  */
1044 static int
1045 tcp_default_handoff_ok(struct tcpcb *tp)
1046 {
1047
1048         return (0);
1049 }
1050
1051 /*
1052  * tfb_tcp_fb_init() function for the default stack.
1053  *
1054  * This handles making sure we have appropriate timers set if you are
1055  * transitioning a socket that has some amount of setup done.
1056  *
1057  * The init() fuction from the default can *never* return non-zero i.e.
1058  * it is required to always succeed since it is the stack of last resort!
1059  */
1060 static int
1061 tcp_default_fb_init(struct tcpcb *tp, void **ptr)
1062 {
1063         struct socket *so = tptosocket(tp);
1064         int rexmt;
1065
1066         INP_WLOCK_ASSERT(tptoinpcb(tp));
1067         /* We don't use the pointer */
1068         *ptr = NULL;
1069
1070         KASSERT(tp->t_state >= 0 && tp->t_state < TCPS_TIME_WAIT,
1071             ("%s: connection %p in unexpected state %d", __func__, tp,
1072             tp->t_state));
1073
1074         /* Make sure we get no interesting mbuf queuing behavior */
1075         /* All mbuf queue/ack compress flags should be off */
1076         tcp_lro_features_off(tp);
1077
1078         /* Cancel the GP measurement in progress */
1079         tp->t_flags &= ~TF_GPUTINPROG;
1080         /* Validate the timers are not in usec, if they are convert */
1081         tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
1082         if ((tp->t_state == TCPS_SYN_SENT) ||
1083             (tp->t_state == TCPS_SYN_RECEIVED))
1084                 rexmt = tcp_rexmit_initial * tcp_backoff[tp->t_rxtshift];
1085         else
1086                 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1087         if (tp->t_rxtshift == 0)
1088                 tp->t_rxtcur = rexmt;
1089         else
1090                 TCPT_RANGESET(tp->t_rxtcur, rexmt, tp->t_rttmin, TCPTV_REXMTMAX);
1091
1092         /*
1093          * Nothing to do for ESTABLISHED or LISTEN states. And, we don't
1094          * know what to do for unexpected states (which includes TIME_WAIT).
1095          */
1096         if (tp->t_state <= TCPS_LISTEN || tp->t_state >= TCPS_TIME_WAIT)
1097                 return (0);
1098
1099         /*
1100          * Make sure some kind of transmission timer is set if there is
1101          * outstanding data.
1102          */
1103         if ((!TCPS_HAVEESTABLISHED(tp->t_state) || sbavail(&so->so_snd) ||
1104             tp->snd_una != tp->snd_max) && !(tcp_timer_active(tp, TT_REXMT) ||
1105             tcp_timer_active(tp, TT_PERSIST))) {
1106                 /*
1107                  * If the session has established and it looks like it should
1108                  * be in the persist state, set the persist timer. Otherwise,
1109                  * set the retransmit timer.
1110                  */
1111                 if (TCPS_HAVEESTABLISHED(tp->t_state) && tp->snd_wnd == 0 &&
1112                     (int32_t)(tp->snd_nxt - tp->snd_una) <
1113                     (int32_t)sbavail(&so->so_snd))
1114                         tcp_setpersist(tp);
1115                 else
1116                         tcp_timer_activate(tp, TT_REXMT, TP_RXTCUR(tp));
1117         }
1118
1119         /* All non-embryonic sessions get a keepalive timer. */
1120         if (!tcp_timer_active(tp, TT_KEEP))
1121                 tcp_timer_activate(tp, TT_KEEP,
1122                     TCPS_HAVEESTABLISHED(tp->t_state) ? TP_KEEPIDLE(tp) :
1123                     TP_KEEPINIT(tp));
1124
1125         /*
1126          * Make sure critical variables are initialized
1127          * if transitioning while in Recovery.
1128          */
1129         if IN_FASTRECOVERY(tp->t_flags) {
1130                 if (tp->sackhint.recover_fs == 0)
1131                         tp->sackhint.recover_fs = max(1,
1132                             tp->snd_nxt - tp->snd_una);
1133         }
1134
1135         return (0);
1136 }
1137
1138 /*
1139  * tfb_tcp_fb_fini() function for the default stack.
1140  *
1141  * This changes state as necessary (or prudent) to prepare for another stack
1142  * to assume responsibility for the connection.
1143  */
1144 static void
1145 tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged)
1146 {
1147
1148         INP_WLOCK_ASSERT(tptoinpcb(tp));
1149
1150 #ifdef TCP_BLACKBOX
1151         tcp_log_flowend(tp);
1152 #endif
1153         tp->t_acktime = 0;
1154         return;
1155 }
1156
1157 /*
1158  * Target size of TCP PCB hash tables. Must be a power of two.
1159  *
1160  * Note that this can be overridden by the kernel environment
1161  * variable net.inet.tcp.tcbhashsize
1162  */
1163 #ifndef TCBHASHSIZE
1164 #define TCBHASHSIZE     0
1165 #endif
1166
1167 MALLOC_DEFINE(M_TCPLOG, "tcplog", "TCP address and flags print buffers");
1168 MALLOC_DEFINE(M_TCPFUNCTIONS, "tcpfunc", "TCP function set memory");
1169
1170 static struct mtx isn_mtx;
1171
1172 #define ISN_LOCK_INIT() mtx_init(&isn_mtx, "isn_mtx", NULL, MTX_DEF)
1173 #define ISN_LOCK()      mtx_lock(&isn_mtx)
1174 #define ISN_UNLOCK()    mtx_unlock(&isn_mtx)
1175
1176 INPCBSTORAGE_DEFINE(tcpcbstor, tcpcb, "tcpinp", "tcp_inpcb", "tcp", "tcphash");
1177
1178 /*
1179  * Take a value and get the next power of 2 that doesn't overflow.
1180  * Used to size the tcp_inpcb hash buckets.
1181  */
1182 static int
1183 maketcp_hashsize(int size)
1184 {
1185         int hashsize;
1186
1187         /*
1188          * auto tune.
1189          * get the next power of 2 higher than maxsockets.
1190          */
1191         hashsize = 1 << fls(size);
1192         /* catch overflow, and just go one power of 2 smaller */
1193         if (hashsize < size) {
1194                 hashsize = 1 << (fls(size) - 1);
1195         }
1196         return (hashsize);
1197 }
1198
1199 static volatile int next_tcp_stack_id = 1;
1200
1201 /*
1202  * Register a TCP function block with the name provided in the names
1203  * array.  (Note that this function does NOT automatically register
1204  * blk->tfb_tcp_block_name as a stack name.  Therefore, you should
1205  * explicitly include blk->tfb_tcp_block_name in the list of names if
1206  * you wish to register the stack with that name.)
1207  *
1208  * Either all name registrations will succeed or all will fail.  If
1209  * a name registration fails, the function will update the num_names
1210  * argument to point to the array index of the name that encountered
1211  * the failure.
1212  *
1213  * Returns 0 on success, or an error code on failure.
1214  */
1215 int
1216 register_tcp_functions_as_names(struct tcp_function_block *blk, int wait,
1217     const char *names[], int *num_names)
1218 {
1219         struct tcp_function *n;
1220         struct tcp_function_set fs;
1221         int error, i;
1222
1223         KASSERT(names != NULL && *num_names > 0,
1224             ("%s: Called with 0-length name list", __func__));
1225         KASSERT(names != NULL, ("%s: Called with NULL name list", __func__));
1226         KASSERT(rw_initialized(&tcp_function_lock),
1227             ("%s: called too early", __func__));
1228
1229         if ((blk->tfb_tcp_output == NULL) ||
1230             (blk->tfb_tcp_do_segment == NULL) ||
1231             (blk->tfb_tcp_ctloutput == NULL) ||
1232             (strlen(blk->tfb_tcp_block_name) == 0)) {
1233                 /*
1234                  * These functions are required and you
1235                  * need a name.
1236                  */
1237                 *num_names = 0;
1238                 return (EINVAL);
1239         }
1240
1241         if (blk->tfb_flags & TCP_FUNC_BEING_REMOVED) {
1242                 *num_names = 0;
1243                 return (EINVAL);
1244         }
1245
1246         refcount_init(&blk->tfb_refcnt, 0);
1247         blk->tfb_id = atomic_fetchadd_int(&next_tcp_stack_id, 1);
1248         for (i = 0; i < *num_names; i++) {
1249                 n = malloc(sizeof(struct tcp_function), M_TCPFUNCTIONS, wait);
1250                 if (n == NULL) {
1251                         error = ENOMEM;
1252                         goto cleanup;
1253                 }
1254                 n->tf_fb = blk;
1255
1256                 (void)strlcpy(fs.function_set_name, names[i],
1257                     sizeof(fs.function_set_name));
1258                 rw_wlock(&tcp_function_lock);
1259                 if (find_tcp_functions_locked(&fs) != NULL) {
1260                         /* Duplicate name space not allowed */
1261                         rw_wunlock(&tcp_function_lock);
1262                         free(n, M_TCPFUNCTIONS);
1263                         error = EALREADY;
1264                         goto cleanup;
1265                 }
1266                 (void)strlcpy(n->tf_name, names[i], sizeof(n->tf_name));
1267                 TAILQ_INSERT_TAIL(&t_functions, n, tf_next);
1268                 tcp_fb_cnt++;
1269                 rw_wunlock(&tcp_function_lock);
1270         }
1271         return(0);
1272
1273 cleanup:
1274         /*
1275          * Deregister the names we just added. Because registration failed
1276          * for names[i], we don't need to deregister that name.
1277          */
1278         *num_names = i;
1279         rw_wlock(&tcp_function_lock);
1280         while (--i >= 0) {
1281                 TAILQ_FOREACH(n, &t_functions, tf_next) {
1282                         if (!strncmp(n->tf_name, names[i],
1283                             TCP_FUNCTION_NAME_LEN_MAX)) {
1284                                 TAILQ_REMOVE(&t_functions, n, tf_next);
1285                                 tcp_fb_cnt--;
1286                                 n->tf_fb = NULL;
1287                                 free(n, M_TCPFUNCTIONS);
1288                                 break;
1289                         }
1290                 }
1291         }
1292         rw_wunlock(&tcp_function_lock);
1293         return (error);
1294 }
1295
1296 /*
1297  * Register a TCP function block using the name provided in the name
1298  * argument.
1299  *
1300  * Returns 0 on success, or an error code on failure.
1301  */
1302 int
1303 register_tcp_functions_as_name(struct tcp_function_block *blk, const char *name,
1304     int wait)
1305 {
1306         const char *name_list[1];
1307         int num_names, rv;
1308
1309         num_names = 1;
1310         if (name != NULL)
1311                 name_list[0] = name;
1312         else
1313                 name_list[0] = blk->tfb_tcp_block_name;
1314         rv = register_tcp_functions_as_names(blk, wait, name_list, &num_names);
1315         return (rv);
1316 }
1317
1318 /*
1319  * Register a TCP function block using the name defined in
1320  * blk->tfb_tcp_block_name.
1321  *
1322  * Returns 0 on success, or an error code on failure.
1323  */
1324 int
1325 register_tcp_functions(struct tcp_function_block *blk, int wait)
1326 {
1327
1328         return (register_tcp_functions_as_name(blk, NULL, wait));
1329 }
1330
1331 /*
1332  * Deregister all names associated with a function block. This
1333  * functionally removes the function block from use within the system.
1334  *
1335  * When called with a true quiesce argument, mark the function block
1336  * as being removed so no more stacks will use it and determine
1337  * whether the removal would succeed.
1338  *
1339  * When called with a false quiesce argument, actually attempt the
1340  * removal.
1341  *
1342  * When called with a force argument, attempt to switch all TCBs to
1343  * use the default stack instead of returning EBUSY.
1344  *
1345  * Returns 0 on success (or if the removal would succeed), or an error
1346  * code on failure.
1347  */
1348 int
1349 deregister_tcp_functions(struct tcp_function_block *blk, bool quiesce,
1350     bool force)
1351 {
1352         struct tcp_function *f;
1353         VNET_ITERATOR_DECL(vnet_iter);
1354
1355         if (blk == &tcp_def_funcblk) {
1356                 /* You can't un-register the default */
1357                 return (EPERM);
1358         }
1359         rw_wlock(&tcp_function_lock);
1360         VNET_LIST_RLOCK_NOSLEEP();
1361         VNET_FOREACH(vnet_iter) {
1362                 CURVNET_SET(vnet_iter);
1363                 if (blk == V_tcp_func_set_ptr) {
1364                         /* You can't free the current default in some vnet. */
1365                         CURVNET_RESTORE();
1366                         VNET_LIST_RUNLOCK_NOSLEEP();
1367                         rw_wunlock(&tcp_function_lock);
1368                         return (EBUSY);
1369                 }
1370                 CURVNET_RESTORE();
1371         }
1372         VNET_LIST_RUNLOCK_NOSLEEP();
1373         /* Mark the block so no more stacks can use it. */
1374         blk->tfb_flags |= TCP_FUNC_BEING_REMOVED;
1375         /*
1376          * If TCBs are still attached to the stack, attempt to switch them
1377          * to the default stack.
1378          */
1379         if (force && blk->tfb_refcnt) {
1380                 struct inpcb *inp;
1381                 struct tcpcb *tp;
1382                 VNET_ITERATOR_DECL(vnet_iter);
1383
1384                 rw_wunlock(&tcp_function_lock);
1385
1386                 VNET_LIST_RLOCK();
1387                 VNET_FOREACH(vnet_iter) {
1388                         CURVNET_SET(vnet_iter);
1389                         struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
1390                             INPLOOKUP_WLOCKPCB);
1391
1392                         while ((inp = inp_next(&inpi)) != NULL) {
1393                                 tp = intotcpcb(inp);
1394                                 if (tp == NULL || tp->t_fb != blk)
1395                                         continue;
1396                                 tcp_switch_back_to_default(tp);
1397                         }
1398                         CURVNET_RESTORE();
1399                 }
1400                 VNET_LIST_RUNLOCK();
1401
1402                 rw_wlock(&tcp_function_lock);
1403         }
1404         if (blk->tfb_refcnt) {
1405                 /* TCBs still attached. */
1406                 rw_wunlock(&tcp_function_lock);
1407                 return (EBUSY);
1408         }
1409         if (quiesce) {
1410                 /* Skip removal. */
1411                 rw_wunlock(&tcp_function_lock);
1412                 return (0);
1413         }
1414         /* Remove any function names that map to this function block. */
1415         while (find_tcp_fb_locked(blk, &f) != NULL) {
1416                 TAILQ_REMOVE(&t_functions, f, tf_next);
1417                 tcp_fb_cnt--;
1418                 f->tf_fb = NULL;
1419                 free(f, M_TCPFUNCTIONS);
1420         }
1421         rw_wunlock(&tcp_function_lock);
1422         return (0);
1423 }
1424
1425 static void
1426 tcp_drain(void)
1427 {
1428         struct epoch_tracker et;
1429         VNET_ITERATOR_DECL(vnet_iter);
1430
1431         if (!do_tcpdrain)
1432                 return;
1433
1434         NET_EPOCH_ENTER(et);
1435         VNET_LIST_RLOCK_NOSLEEP();
1436         VNET_FOREACH(vnet_iter) {
1437                 CURVNET_SET(vnet_iter);
1438                 struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
1439                     INPLOOKUP_WLOCKPCB);
1440                 struct inpcb *inpb;
1441                 struct tcpcb *tcpb;
1442
1443         /*
1444          * Walk the tcpbs, if existing, and flush the reassembly queue,
1445          * if there is one...
1446          * XXX: The "Net/3" implementation doesn't imply that the TCP
1447          *      reassembly queue should be flushed, but in a situation
1448          *      where we're really low on mbufs, this is potentially
1449          *      useful.
1450          */
1451                 while ((inpb = inp_next(&inpi)) != NULL) {
1452                         if ((tcpb = intotcpcb(inpb)) != NULL) {
1453                                 tcp_reass_flush(tcpb);
1454                                 tcp_clean_sackreport(tcpb);
1455 #ifdef TCP_BLACKBOX
1456                                 tcp_log_drain(tcpb);
1457 #endif
1458 #ifdef TCPPCAP
1459                                 if (tcp_pcap_aggressive_free) {
1460                                         /* Free the TCP PCAP queues. */
1461                                         tcp_pcap_drain(&(tcpb->t_inpkts));
1462                                         tcp_pcap_drain(&(tcpb->t_outpkts));
1463                                 }
1464 #endif
1465                         }
1466                 }
1467                 CURVNET_RESTORE();
1468         }
1469         VNET_LIST_RUNLOCK_NOSLEEP();
1470         NET_EPOCH_EXIT(et);
1471 }
1472
1473 static void
1474 tcp_vnet_init(void *arg __unused)
1475 {
1476
1477 #ifdef TCP_HHOOK
1478         if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN,
1479             &V_tcp_hhh[HHOOK_TCP_EST_IN], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
1480                 printf("%s: WARNING: unable to register helper hook\n", __func__);
1481         if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT,
1482             &V_tcp_hhh[HHOOK_TCP_EST_OUT], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
1483                 printf("%s: WARNING: unable to register helper hook\n", __func__);
1484 #endif
1485 #ifdef STATS
1486         if (tcp_stats_init())
1487                 printf("%s: WARNING: unable to initialise TCP stats\n",
1488                     __func__);
1489 #endif
1490         in_pcbinfo_init(&V_tcbinfo, &tcpcbstor, tcp_tcbhashsize,
1491             tcp_tcbhashsize);
1492
1493         syncache_init();
1494         tcp_hc_init();
1495
1496         TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack);
1497         V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
1498             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1499
1500         tcp_fastopen_init();
1501
1502         COUNTER_ARRAY_ALLOC(V_tcps_states, TCP_NSTATES, M_WAITOK);
1503         VNET_PCPUSTAT_ALLOC(tcpstat, M_WAITOK);
1504
1505         V_tcp_msl = TCPTV_MSL;
1506 }
1507 VNET_SYSINIT(tcp_vnet_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH,
1508     tcp_vnet_init, NULL);
1509
1510 static void
1511 tcp_init(void *arg __unused)
1512 {
1513         const char *tcbhash_tuneable;
1514         int hashsize;
1515
1516         tcp_reass_global_init();
1517
1518         /* XXX virtualize those below? */
1519         tcp_delacktime = TCPTV_DELACK;
1520         tcp_keepinit = TCPTV_KEEP_INIT;
1521         tcp_keepidle = TCPTV_KEEP_IDLE;
1522         tcp_keepintvl = TCPTV_KEEPINTVL;
1523         tcp_maxpersistidle = TCPTV_KEEP_IDLE;
1524         tcp_rexmit_initial = TCPTV_RTOBASE;
1525         if (tcp_rexmit_initial < 1)
1526                 tcp_rexmit_initial = 1;
1527         tcp_rexmit_min = TCPTV_MIN;
1528         if (tcp_rexmit_min < 1)
1529                 tcp_rexmit_min = 1;
1530         tcp_persmin = TCPTV_PERSMIN;
1531         tcp_persmax = TCPTV_PERSMAX;
1532         tcp_rexmit_slop = TCPTV_CPU_VAR;
1533         tcp_finwait2_timeout = TCPTV_FINWAIT2_TIMEOUT;
1534
1535         /* Setup the tcp function block list */
1536         TAILQ_INIT(&t_functions);
1537         rw_init(&tcp_function_lock, "tcp_func_lock");
1538         register_tcp_functions(&tcp_def_funcblk, M_WAITOK);
1539         sx_init(&tcpoudp_lock, "TCP over UDP configuration");
1540 #ifdef TCP_BLACKBOX
1541         /* Initialize the TCP logging data. */
1542         tcp_log_init();
1543 #endif
1544         arc4rand(&V_ts_offset_secret, sizeof(V_ts_offset_secret), 0);
1545
1546         if (tcp_soreceive_stream) {
1547 #ifdef INET
1548                 tcp_protosw.pr_soreceive = soreceive_stream;
1549 #endif
1550 #ifdef INET6
1551                 tcp6_protosw.pr_soreceive = soreceive_stream;
1552 #endif /* INET6 */
1553         }
1554
1555 #ifdef INET6
1556         max_protohdr_grow(sizeof(struct ip6_hdr) + sizeof(struct tcphdr));
1557 #else /* INET6 */
1558         max_protohdr_grow(sizeof(struct tcpiphdr));
1559 #endif /* INET6 */
1560
1561         ISN_LOCK_INIT();
1562         EVENTHANDLER_REGISTER(shutdown_pre_sync, tcp_fini, NULL,
1563                 SHUTDOWN_PRI_DEFAULT);
1564         EVENTHANDLER_REGISTER(vm_lowmem, tcp_drain, NULL, LOWMEM_PRI_DEFAULT);
1565         EVENTHANDLER_REGISTER(mbuf_lowmem, tcp_drain, NULL, LOWMEM_PRI_DEFAULT);
1566
1567         tcp_inp_lro_direct_queue = counter_u64_alloc(M_WAITOK);
1568         tcp_inp_lro_wokeup_queue = counter_u64_alloc(M_WAITOK);
1569         tcp_inp_lro_compressed = counter_u64_alloc(M_WAITOK);
1570         tcp_inp_lro_locks_taken = counter_u64_alloc(M_WAITOK);
1571         tcp_extra_mbuf = counter_u64_alloc(M_WAITOK);
1572         tcp_would_have_but = counter_u64_alloc(M_WAITOK);
1573         tcp_comp_total = counter_u64_alloc(M_WAITOK);
1574         tcp_uncomp_total = counter_u64_alloc(M_WAITOK);
1575         tcp_bad_csums = counter_u64_alloc(M_WAITOK);
1576         tcp_pacing_failures = counter_u64_alloc(M_WAITOK);
1577 #ifdef TCPPCAP
1578         tcp_pcap_init();
1579 #endif
1580
1581         hashsize = TCBHASHSIZE;
1582         tcbhash_tuneable = "net.inet.tcp.tcbhashsize";
1583         TUNABLE_INT_FETCH(tcbhash_tuneable, &hashsize);
1584         if (hashsize == 0) {
1585                 /*
1586                  * Auto tune the hash size based on maxsockets.
1587                  * A perfect hash would have a 1:1 mapping
1588                  * (hashsize = maxsockets) however it's been
1589                  * suggested that O(2) average is better.
1590                  */
1591                 hashsize = maketcp_hashsize(maxsockets / 4);
1592                 /*
1593                  * Our historical default is 512,
1594                  * do not autotune lower than this.
1595                  */
1596                 if (hashsize < 512)
1597                         hashsize = 512;
1598                 if (bootverbose)
1599                         printf("%s: %s auto tuned to %d\n", __func__,
1600                             tcbhash_tuneable, hashsize);
1601         }
1602         /*
1603          * We require a hashsize to be a power of two.
1604          * Previously if it was not a power of two we would just reset it
1605          * back to 512, which could be a nasty surprise if you did not notice
1606          * the error message.
1607          * Instead what we do is clip it to the closest power of two lower
1608          * than the specified hash value.
1609          */
1610         if (!powerof2(hashsize)) {
1611                 int oldhashsize = hashsize;
1612
1613                 hashsize = maketcp_hashsize(hashsize);
1614                 /* prevent absurdly low value */
1615                 if (hashsize < 16)
1616                         hashsize = 16;
1617                 printf("%s: WARNING: TCB hash size not a power of 2, "
1618                     "clipped from %d to %d.\n", __func__, oldhashsize,
1619                     hashsize);
1620         }
1621         tcp_tcbhashsize = hashsize;
1622
1623 #ifdef INET
1624         IPPROTO_REGISTER(IPPROTO_TCP, tcp_input, tcp_ctlinput);
1625 #endif
1626 #ifdef INET6
1627         IP6PROTO_REGISTER(IPPROTO_TCP, tcp6_input, tcp6_ctlinput);
1628 #endif
1629 }
1630 SYSINIT(tcp_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, tcp_init, NULL);
1631
1632 #ifdef VIMAGE
1633 static void
1634 tcp_destroy(void *unused __unused)
1635 {
1636         int n;
1637 #ifdef TCP_HHOOK
1638         int error;
1639 #endif
1640
1641         /*
1642          * All our processes are gone, all our sockets should be cleaned
1643          * up, which means, we should be past the tcp_discardcb() calls.
1644          * Sleep to let all tcpcb timers really disappear and cleanup.
1645          */
1646         for (;;) {
1647                 INP_INFO_WLOCK(&V_tcbinfo);
1648                 n = V_tcbinfo.ipi_count;
1649                 INP_INFO_WUNLOCK(&V_tcbinfo);
1650                 if (n == 0)
1651                         break;
1652                 pause("tcpdes", hz / 10);
1653         }
1654         tcp_hc_destroy();
1655         syncache_destroy();
1656         in_pcbinfo_destroy(&V_tcbinfo);
1657         /* tcp_discardcb() clears the sack_holes up. */
1658         uma_zdestroy(V_sack_hole_zone);
1659
1660         /*
1661          * Cannot free the zone until all tcpcbs are released as we attach
1662          * the allocations to them.
1663          */
1664         tcp_fastopen_destroy();
1665
1666         COUNTER_ARRAY_FREE(V_tcps_states, TCP_NSTATES);
1667         VNET_PCPUSTAT_FREE(tcpstat);
1668
1669 #ifdef TCP_HHOOK
1670         error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_IN]);
1671         if (error != 0) {
1672                 printf("%s: WARNING: unable to deregister helper hook "
1673                     "type=%d, id=%d: error %d returned\n", __func__,
1674                     HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN, error);
1675         }
1676         error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_OUT]);
1677         if (error != 0) {
1678                 printf("%s: WARNING: unable to deregister helper hook "
1679                     "type=%d, id=%d: error %d returned\n", __func__,
1680                     HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT, error);
1681         }
1682 #endif
1683 }
1684 VNET_SYSUNINIT(tcp, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH, tcp_destroy, NULL);
1685 #endif
1686
1687 void
1688 tcp_fini(void *xtp)
1689 {
1690
1691 }
1692
1693 /*
1694  * Fill in the IP and TCP headers for an outgoing packet, given the tcpcb.
1695  * tcp_template used to store this data in mbufs, but we now recopy it out
1696  * of the tcpcb each time to conserve mbufs.
1697  */
1698 void
1699 tcpip_fillheaders(struct inpcb *inp, uint16_t port, void *ip_ptr, void *tcp_ptr)
1700 {
1701         struct tcphdr *th = (struct tcphdr *)tcp_ptr;
1702
1703         INP_WLOCK_ASSERT(inp);
1704
1705 #ifdef INET6
1706         if ((inp->inp_vflag & INP_IPV6) != 0) {
1707                 struct ip6_hdr *ip6;
1708
1709                 ip6 = (struct ip6_hdr *)ip_ptr;
1710                 ip6->ip6_flow = (ip6->ip6_flow & ~IPV6_FLOWINFO_MASK) |
1711                         (inp->inp_flow & IPV6_FLOWINFO_MASK);
1712                 ip6->ip6_vfc = (ip6->ip6_vfc & ~IPV6_VERSION_MASK) |
1713                         (IPV6_VERSION & IPV6_VERSION_MASK);
1714                 if (port == 0)
1715                         ip6->ip6_nxt = IPPROTO_TCP;
1716                 else
1717                         ip6->ip6_nxt = IPPROTO_UDP;
1718                 ip6->ip6_plen = htons(sizeof(struct tcphdr));
1719                 ip6->ip6_src = inp->in6p_laddr;
1720                 ip6->ip6_dst = inp->in6p_faddr;
1721         }
1722 #endif /* INET6 */
1723 #if defined(INET6) && defined(INET)
1724         else
1725 #endif
1726 #ifdef INET
1727         {
1728                 struct ip *ip;
1729
1730                 ip = (struct ip *)ip_ptr;
1731                 ip->ip_v = IPVERSION;
1732                 ip->ip_hl = 5;
1733                 ip->ip_tos = inp->inp_ip_tos;
1734                 ip->ip_len = 0;
1735                 ip->ip_id = 0;
1736                 ip->ip_off = 0;
1737                 ip->ip_ttl = inp->inp_ip_ttl;
1738                 ip->ip_sum = 0;
1739                 if (port == 0)
1740                         ip->ip_p = IPPROTO_TCP;
1741                 else
1742                         ip->ip_p = IPPROTO_UDP;
1743                 ip->ip_src = inp->inp_laddr;
1744                 ip->ip_dst = inp->inp_faddr;
1745         }
1746 #endif /* INET */
1747         th->th_sport = inp->inp_lport;
1748         th->th_dport = inp->inp_fport;
1749         th->th_seq = 0;
1750         th->th_ack = 0;
1751         th->th_off = 5;
1752         tcp_set_flags(th, 0);
1753         th->th_win = 0;
1754         th->th_urp = 0;
1755         th->th_sum = 0;         /* in_pseudo() is called later for ipv4 */
1756 }
1757
1758 /*
1759  * Create template to be used to send tcp packets on a connection.
1760  * Allocates an mbuf and fills in a skeletal tcp/ip header.  The only
1761  * use for this function is in keepalives, which use tcp_respond.
1762  */
1763 struct tcptemp *
1764 tcpip_maketemplate(struct inpcb *inp)
1765 {
1766         struct tcptemp *t;
1767
1768         t = malloc(sizeof(*t), M_TEMP, M_NOWAIT);
1769         if (t == NULL)
1770                 return (NULL);
1771         tcpip_fillheaders(inp, 0, (void *)&t->tt_ipgen, (void *)&t->tt_t);
1772         return (t);
1773 }
1774
1775 /*
1776  * Send a single message to the TCP at address specified by
1777  * the given TCP/IP header.  If m == NULL, then we make a copy
1778  * of the tcpiphdr at th and send directly to the addressed host.
1779  * This is used to force keep alive messages out using the TCP
1780  * template for a connection.  If flags are given then we send
1781  * a message back to the TCP which originated the segment th,
1782  * and discard the mbuf containing it and any other attached mbufs.
1783  *
1784  * In any case the ack and sequence number of the transmitted
1785  * segment are as specified by the parameters.
1786  *
1787  * NOTE: If m != NULL, then th must point to *inside* the mbuf.
1788  */
1789 void
1790 tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m,
1791     tcp_seq ack, tcp_seq seq, uint16_t flags)
1792 {
1793         struct tcpopt to;
1794         struct inpcb *inp;
1795         struct ip *ip;
1796         struct mbuf *optm;
1797         struct udphdr *uh = NULL;
1798         struct tcphdr *nth;
1799         struct tcp_log_buffer *lgb;
1800         u_char *optp;
1801 #ifdef INET6
1802         struct ip6_hdr *ip6;
1803         int isipv6;
1804 #endif /* INET6 */
1805         int optlen, tlen, win, ulen;
1806         int ect = 0;
1807         bool incl_opts;
1808         uint16_t port;
1809         int output_ret;
1810 #ifdef INVARIANTS
1811         int thflags = tcp_get_flags(th);
1812 #endif
1813
1814         KASSERT(tp != NULL || m != NULL, ("tcp_respond: tp and m both NULL"));
1815         NET_EPOCH_ASSERT();
1816
1817 #ifdef INET6
1818         isipv6 = ((struct ip *)ipgen)->ip_v == (IPV6_VERSION >> 4);
1819         ip6 = ipgen;
1820 #endif /* INET6 */
1821         ip = ipgen;
1822
1823         if (tp != NULL) {
1824                 inp = tptoinpcb(tp);
1825                 INP_LOCK_ASSERT(inp);
1826         } else
1827                 inp = NULL;
1828
1829         if (m != NULL) {
1830 #ifdef INET6
1831                 if (isipv6 && ip6 && (ip6->ip6_nxt == IPPROTO_UDP))
1832                         port = m->m_pkthdr.tcp_tun_port;
1833                 else
1834 #endif
1835                 if (ip && (ip->ip_p == IPPROTO_UDP))
1836                         port = m->m_pkthdr.tcp_tun_port;
1837                 else
1838                         port = 0;
1839         } else
1840                 port = tp->t_port;
1841
1842         incl_opts = false;
1843         win = 0;
1844         if (tp != NULL) {
1845                 if (!(flags & TH_RST)) {
1846                         win = sbspace(&inp->inp_socket->so_rcv);
1847                         if (win > TCP_MAXWIN << tp->rcv_scale)
1848                                 win = TCP_MAXWIN << tp->rcv_scale;
1849                 }
1850                 if ((tp->t_flags & TF_NOOPT) == 0)
1851                         incl_opts = true;
1852         }
1853         if (m == NULL) {
1854                 m = m_gethdr(M_NOWAIT, MT_DATA);
1855                 if (m == NULL)
1856                         return;
1857                 m->m_data += max_linkhdr;
1858 #ifdef INET6
1859                 if (isipv6) {
1860                         bcopy((caddr_t)ip6, mtod(m, caddr_t),
1861                               sizeof(struct ip6_hdr));
1862                         ip6 = mtod(m, struct ip6_hdr *);
1863                         nth = (struct tcphdr *)(ip6 + 1);
1864                         if (port) {
1865                                 /* Insert a UDP header */
1866                                 uh = (struct udphdr *)nth;
1867                                 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1868                                 uh->uh_dport = port;
1869                                 nth = (struct tcphdr *)(uh + 1);
1870                         }
1871                 } else
1872 #endif /* INET6 */
1873                 {
1874                         bcopy((caddr_t)ip, mtod(m, caddr_t), sizeof(struct ip));
1875                         ip = mtod(m, struct ip *);
1876                         nth = (struct tcphdr *)(ip + 1);
1877                         if (port) {
1878                                 /* Insert a UDP header */
1879                                 uh = (struct udphdr *)nth;
1880                                 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1881                                 uh->uh_dport = port;
1882                                 nth = (struct tcphdr *)(uh + 1);
1883                         }
1884                 }
1885                 bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
1886                 flags = TH_ACK;
1887         } else if ((!M_WRITABLE(m)) || (port != 0)) {
1888                 struct mbuf *n;
1889
1890                 /* Can't reuse 'm', allocate a new mbuf. */
1891                 n = m_gethdr(M_NOWAIT, MT_DATA);
1892                 if (n == NULL) {
1893                         m_freem(m);
1894                         return;
1895                 }
1896
1897                 if (!m_dup_pkthdr(n, m, M_NOWAIT)) {
1898                         m_freem(m);
1899                         m_freem(n);
1900                         return;
1901                 }
1902
1903                 n->m_data += max_linkhdr;
1904                 /* m_len is set later */
1905 #define xchg(a,b,type) { type t; t=a; a=b; b=t; }
1906 #ifdef INET6
1907                 if (isipv6) {
1908                         bcopy((caddr_t)ip6, mtod(n, caddr_t),
1909                               sizeof(struct ip6_hdr));
1910                         ip6 = mtod(n, struct ip6_hdr *);
1911                         xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
1912                         nth = (struct tcphdr *)(ip6 + 1);
1913                         if (port) {
1914                                 /* Insert a UDP header */
1915                                 uh = (struct udphdr *)nth;
1916                                 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1917                                 uh->uh_dport = port;
1918                                 nth = (struct tcphdr *)(uh + 1);
1919                         }
1920                 } else
1921 #endif /* INET6 */
1922                 {
1923                         bcopy((caddr_t)ip, mtod(n, caddr_t), sizeof(struct ip));
1924                         ip = mtod(n, struct ip *);
1925                         xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
1926                         nth = (struct tcphdr *)(ip + 1);
1927                         if (port) {
1928                                 /* Insert a UDP header */
1929                                 uh = (struct udphdr *)nth;
1930                                 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1931                                 uh->uh_dport = port;
1932                                 nth = (struct tcphdr *)(uh + 1);
1933                         }
1934                 }
1935                 bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
1936                 xchg(nth->th_dport, nth->th_sport, uint16_t);
1937                 th = nth;
1938                 m_freem(m);
1939                 m = n;
1940         } else {
1941                 /*
1942                  *  reuse the mbuf.
1943                  * XXX MRT We inherit the FIB, which is lucky.
1944                  */
1945                 m_freem(m->m_next);
1946                 m->m_next = NULL;
1947                 m->m_data = (caddr_t)ipgen;
1948                 /* clear any receive flags for proper bpf timestamping */
1949                 m->m_flags &= ~(M_TSTMP | M_TSTMP_LRO);
1950                 /* m_len is set later */
1951 #ifdef INET6
1952                 if (isipv6) {
1953                         xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
1954                         nth = (struct tcphdr *)(ip6 + 1);
1955                 } else
1956 #endif /* INET6 */
1957                 {
1958                         xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
1959                         nth = (struct tcphdr *)(ip + 1);
1960                 }
1961                 if (th != nth) {
1962                         /*
1963                          * this is usually a case when an extension header
1964                          * exists between the IPv6 header and the
1965                          * TCP header.
1966                          */
1967                         nth->th_sport = th->th_sport;
1968                         nth->th_dport = th->th_dport;
1969                 }
1970                 xchg(nth->th_dport, nth->th_sport, uint16_t);
1971 #undef xchg
1972         }
1973         tlen = 0;
1974 #ifdef INET6
1975         if (isipv6)
1976                 tlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
1977 #endif
1978 #if defined(INET) && defined(INET6)
1979         else
1980 #endif
1981 #ifdef INET
1982                 tlen = sizeof (struct tcpiphdr);
1983 #endif
1984         if (port)
1985                 tlen += sizeof (struct udphdr);
1986 #ifdef INVARIANTS
1987         m->m_len = 0;
1988         KASSERT(M_TRAILINGSPACE(m) >= tlen,
1989             ("Not enough trailing space for message (m=%p, need=%d, have=%ld)",
1990             m, tlen, (long)M_TRAILINGSPACE(m)));
1991 #endif
1992         m->m_len = tlen;
1993         to.to_flags = 0;
1994         if (incl_opts) {
1995                 ect = tcp_ecn_output_established(tp, &flags, 0, false);
1996                 /* Make sure we have room. */
1997                 if (M_TRAILINGSPACE(m) < TCP_MAXOLEN) {
1998                         m->m_next = m_get(M_NOWAIT, MT_DATA);
1999                         if (m->m_next) {
2000                                 optp = mtod(m->m_next, u_char *);
2001                                 optm = m->m_next;
2002                         } else
2003                                 incl_opts = false;
2004                 } else {
2005                         optp = (u_char *) (nth + 1);
2006                         optm = m;
2007                 }
2008         }
2009         if (incl_opts) {
2010                 /* Timestamps. */
2011                 if (tp->t_flags & TF_RCVD_TSTMP) {
2012                         to.to_tsval = tcp_ts_getticks() + tp->ts_offset;
2013                         to.to_tsecr = tp->ts_recent;
2014                         to.to_flags |= TOF_TS;
2015                 }
2016 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2017                 /* TCP-MD5 (RFC2385). */
2018                 if (tp->t_flags & TF_SIGNATURE)
2019                         to.to_flags |= TOF_SIGNATURE;
2020 #endif
2021                 /* Add the options. */
2022                 tlen += optlen = tcp_addoptions(&to, optp);
2023
2024                 /* Update m_len in the correct mbuf. */
2025                 optm->m_len += optlen;
2026         } else
2027                 optlen = 0;
2028 #ifdef INET6
2029         if (isipv6) {
2030                 if (uh) {
2031                         ulen = tlen - sizeof(struct ip6_hdr);
2032                         uh->uh_ulen = htons(ulen);
2033                 }
2034                 ip6->ip6_flow = htonl(ect << IPV6_FLOWLABEL_LEN);
2035                 ip6->ip6_vfc = IPV6_VERSION;
2036                 if (port)
2037                         ip6->ip6_nxt = IPPROTO_UDP;
2038                 else
2039                         ip6->ip6_nxt = IPPROTO_TCP;
2040                 ip6->ip6_plen = htons(tlen - sizeof(*ip6));
2041         }
2042 #endif
2043 #if defined(INET) && defined(INET6)
2044         else
2045 #endif
2046 #ifdef INET
2047         {
2048                 if (uh) {
2049                         ulen = tlen - sizeof(struct ip);
2050                         uh->uh_ulen = htons(ulen);
2051                 }
2052                 ip->ip_len = htons(tlen);
2053                 if (inp != NULL) {
2054                         ip->ip_tos = inp->inp_ip_tos & ~IPTOS_ECN_MASK;
2055                         ip->ip_ttl = inp->inp_ip_ttl;
2056                 } else {
2057                         ip->ip_tos = 0;
2058                         ip->ip_ttl = V_ip_defttl;
2059                 }
2060                 ip->ip_tos |= ect;
2061                 if (port) {
2062                         ip->ip_p = IPPROTO_UDP;
2063                 } else {
2064                         ip->ip_p = IPPROTO_TCP;
2065                 }
2066                 if (V_path_mtu_discovery)
2067                         ip->ip_off |= htons(IP_DF);
2068         }
2069 #endif
2070         m->m_pkthdr.len = tlen;
2071         m->m_pkthdr.rcvif = NULL;
2072 #ifdef MAC
2073         if (inp != NULL) {
2074                 /*
2075                  * Packet is associated with a socket, so allow the
2076                  * label of the response to reflect the socket label.
2077                  */
2078                 INP_LOCK_ASSERT(inp);
2079                 mac_inpcb_create_mbuf(inp, m);
2080         } else {
2081                 /*
2082                  * Packet is not associated with a socket, so possibly
2083                  * update the label in place.
2084                  */
2085                 mac_netinet_tcp_reply(m);
2086         }
2087 #endif
2088         nth->th_seq = htonl(seq);
2089         nth->th_ack = htonl(ack);
2090         nth->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
2091         tcp_set_flags(nth, flags);
2092         if (tp && (flags & TH_RST)) {
2093                 /* Log the reset */
2094                 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
2095         }
2096         if (tp != NULL)
2097                 nth->th_win = htons((u_short) (win >> tp->rcv_scale));
2098         else
2099                 nth->th_win = htons((u_short)win);
2100         nth->th_urp = 0;
2101
2102 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2103         if (to.to_flags & TOF_SIGNATURE) {
2104                 if (!TCPMD5_ENABLED() ||
2105                     TCPMD5_OUTPUT(m, nth, to.to_signature) != 0) {
2106                         m_freem(m);
2107                         return;
2108                 }
2109         }
2110 #endif
2111
2112 #ifdef INET6
2113         if (isipv6) {
2114                 if (port) {
2115                         m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
2116                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2117                         uh->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
2118                         nth->th_sum = 0;
2119                 } else {
2120                         m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
2121                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2122                         nth->th_sum = in6_cksum_pseudo(ip6,
2123                             tlen - sizeof(struct ip6_hdr), IPPROTO_TCP, 0);
2124                 }
2125                 ip6->ip6_hlim = in6_selecthlim(inp, NULL);
2126         }
2127 #endif /* INET6 */
2128 #if defined(INET6) && defined(INET)
2129         else
2130 #endif
2131 #ifdef INET
2132         {
2133                 if (port) {
2134                         uh->uh_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2135                             htons(ulen + IPPROTO_UDP));
2136                         m->m_pkthdr.csum_flags = CSUM_UDP;
2137                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2138                         nth->th_sum = 0;
2139                 } else {
2140                         m->m_pkthdr.csum_flags = CSUM_TCP;
2141                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2142                         nth->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2143                             htons((u_short)(tlen - sizeof(struct ip) + ip->ip_p)));
2144                 }
2145         }
2146 #endif /* INET */
2147         TCP_PROBE3(debug__output, tp, th, m);
2148         if (flags & TH_RST)
2149                 TCP_PROBE5(accept__refused, NULL, NULL, m, tp, nth);
2150         lgb = NULL;
2151         if ((tp != NULL) && tcp_bblogging_on(tp)) {
2152                 if (INP_WLOCKED(inp)) {
2153                         union tcp_log_stackspecific log;
2154                         struct timeval tv;
2155
2156                         memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2157                         log.u_bbr.inhpts = tcp_in_hpts(tp);
2158                         log.u_bbr.flex8 = 4;
2159                         log.u_bbr.pkts_out = tp->t_maxseg;
2160                         log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2161                         log.u_bbr.delivered = 0;
2162                         lgb = tcp_log_event(tp, nth, NULL, NULL, TCP_LOG_OUT,
2163                             ERRNO_UNK, 0, &log, false, NULL, NULL, 0, &tv);
2164                 } else {
2165                         /*
2166                          * We can not log the packet, since we only own the
2167                          * read lock, but a write lock is needed. The read lock
2168                          * is not upgraded to a write lock, since only getting
2169                          * the read lock was done intentionally to improve the
2170                          * handling of SYN flooding attacks.
2171                          * This happens only for pure SYN segments received in
2172                          * the initial CLOSED state, or received in a more
2173                          * advanced state than listen and the UDP encapsulation
2174                          * port is unexpected.
2175                          * The incoming SYN segments do not really belong to
2176                          * the TCP connection and the handling does not change
2177                          * the state of the TCP connection. Therefore, the
2178                          * sending of the RST segments is not logged. Please
2179                          * note that also the incoming SYN segments are not
2180                          * logged.
2181                          *
2182                          * The following code ensures that the above description
2183                          * is and stays correct.
2184                          */
2185                         KASSERT((thflags & (TH_ACK|TH_SYN)) == TH_SYN &&
2186                             (tp->t_state == TCPS_CLOSED ||
2187                             (tp->t_state > TCPS_LISTEN && tp->t_port != port)),
2188                             ("%s: Logging of TCP segment with flags 0x%b and "
2189                             "UDP encapsulation port %u skipped in state %s",
2190                             __func__, thflags, PRINT_TH_FLAGS,
2191                             ntohs(port), tcpstates[tp->t_state]));
2192                 }
2193         }
2194
2195         if (flags & TH_ACK)
2196                 TCPSTAT_INC(tcps_sndacks);
2197         else if (flags & (TH_SYN|TH_FIN|TH_RST))
2198                 TCPSTAT_INC(tcps_sndctrl);
2199         TCPSTAT_INC(tcps_sndtotal);
2200
2201 #ifdef INET6
2202         if (isipv6) {
2203                 TCP_PROBE5(send, NULL, tp, ip6, tp, nth);
2204                 output_ret = ip6_output(m, inp ? inp->in6p_outputopts : NULL,
2205                     NULL, 0, NULL, NULL, inp);
2206         }
2207 #endif /* INET6 */
2208 #if defined(INET) && defined(INET6)
2209         else
2210 #endif
2211 #ifdef INET
2212         {
2213                 TCP_PROBE5(send, NULL, tp, ip, tp, nth);
2214                 output_ret = ip_output(m, NULL, NULL, 0, NULL, inp);
2215         }
2216 #endif
2217         if (lgb != NULL)
2218                 lgb->tlb_errno = output_ret;
2219 }
2220
2221 /*
2222  * Create a new TCP control block, making an empty reassembly queue and hooking
2223  * it to the argument protocol control block.  The `inp' parameter must have
2224  * come from the zone allocator set up by tcpcbstor declaration.
2225  */
2226 struct tcpcb *
2227 tcp_newtcpcb(struct inpcb *inp)
2228 {
2229         struct tcpcb *tp = intotcpcb(inp);
2230 #ifdef INET6
2231         int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
2232 #endif /* INET6 */
2233
2234         /*
2235          * Historically allocation was done with M_ZERO.  There is a lot of
2236          * code that rely on that.  For now take safe approach and zero whole
2237          * tcpcb.  This definitely can be optimized.
2238          */
2239         bzero(&tp->t_start_zero, t_zero_size);
2240
2241         /* Initialise cc_var struct for this tcpcb. */
2242         tp->t_ccv.type = IPPROTO_TCP;
2243         tp->t_ccv.ccvc.tcp = tp;
2244         rw_rlock(&tcp_function_lock);
2245         tp->t_fb = V_tcp_func_set_ptr;
2246         refcount_acquire(&tp->t_fb->tfb_refcnt);
2247         rw_runlock(&tcp_function_lock);
2248         /*
2249          * Use the current system default CC algorithm.
2250          */
2251         cc_attach(tp, CC_DEFAULT_ALGO());
2252
2253         if (CC_ALGO(tp)->cb_init != NULL)
2254                 if (CC_ALGO(tp)->cb_init(&tp->t_ccv, NULL) > 0) {
2255                         cc_detach(tp);
2256                         if (tp->t_fb->tfb_tcp_fb_fini)
2257                                 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2258                         refcount_release(&tp->t_fb->tfb_refcnt);
2259                         return (NULL);
2260                 }
2261
2262 #ifdef TCP_HHOOK
2263         if (khelp_init_osd(HELPER_CLASS_TCP, &tp->t_osd)) {
2264                 if (tp->t_fb->tfb_tcp_fb_fini)
2265                         (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2266                 refcount_release(&tp->t_fb->tfb_refcnt);
2267                 return (NULL);
2268         }
2269 #endif
2270
2271         TAILQ_INIT(&tp->t_segq);
2272         STAILQ_INIT(&tp->t_inqueue);
2273         tp->t_maxseg =
2274 #ifdef INET6
2275                 isipv6 ? V_tcp_v6mssdflt :
2276 #endif /* INET6 */
2277                 V_tcp_mssdflt;
2278
2279         /* All mbuf queue/ack compress flags should be off */
2280         tcp_lro_features_off(tp);
2281
2282         callout_init_rw(&tp->t_callout, &inp->inp_lock, CALLOUT_RETURNUNLOCKED);
2283         for (int i = 0; i < TT_N; i++)
2284                 tp->t_timers[i] = SBT_MAX;
2285
2286         switch (V_tcp_do_rfc1323) {
2287                 case 0:
2288                         break;
2289                 default:
2290                 case 1:
2291                         tp->t_flags = (TF_REQ_SCALE|TF_REQ_TSTMP);
2292                         break;
2293                 case 2:
2294                         tp->t_flags = TF_REQ_SCALE;
2295                         break;
2296                 case 3:
2297                         tp->t_flags = TF_REQ_TSTMP;
2298                         break;
2299         }
2300         if (V_tcp_do_sack)
2301                 tp->t_flags |= TF_SACK_PERMIT;
2302         TAILQ_INIT(&tp->snd_holes);
2303
2304         /*
2305          * Init srtt to TCPTV_SRTTBASE (0), so we can tell that we have no
2306          * rtt estimate.  Set rttvar so that srtt + 4 * rttvar gives
2307          * reasonable initial retransmit time.
2308          */
2309         tp->t_srtt = TCPTV_SRTTBASE;
2310         tp->t_rttvar = ((tcp_rexmit_initial - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
2311         tp->t_rttmin = tcp_rexmit_min;
2312         tp->t_rxtcur = tcp_rexmit_initial;
2313         tp->snd_cwnd = TCP_MAXWIN << TCP_MAX_WINSHIFT;
2314         tp->snd_ssthresh = TCP_MAXWIN << TCP_MAX_WINSHIFT;
2315         tp->t_rcvtime = ticks;
2316         /* We always start with ticks granularity */
2317         tp->t_tmr_granularity = TCP_TMR_GRANULARITY_TICKS;
2318         /*
2319          * IPv4 TTL initialization is necessary for an IPv6 socket as well,
2320          * because the socket may be bound to an IPv6 wildcard address,
2321          * which may match an IPv4-mapped IPv6 address.
2322          */
2323         inp->inp_ip_ttl = V_ip_defttl;
2324 #ifdef TCPHPTS
2325         tcp_hpts_init(tp);
2326 #endif
2327 #ifdef TCPPCAP
2328         /*
2329          * Init the TCP PCAP queues.
2330          */
2331         tcp_pcap_tcpcb_init(tp);
2332 #endif
2333 #ifdef TCP_BLACKBOX
2334         /* Initialize the per-TCPCB log data. */
2335         tcp_log_tcpcbinit(tp);
2336 #endif
2337         tp->t_pacing_rate = -1;
2338         if (tp->t_fb->tfb_tcp_fb_init) {
2339                 if ((*tp->t_fb->tfb_tcp_fb_init)(tp, &tp->t_fb_ptr)) {
2340                         refcount_release(&tp->t_fb->tfb_refcnt);
2341                         return (NULL);
2342                 }
2343         }
2344 #ifdef STATS
2345         if (V_tcp_perconn_stats_enable == 1)
2346                 tp->t_stats = stats_blob_alloc(V_tcp_perconn_stats_dflt_tpl, 0);
2347 #endif
2348         if (V_tcp_do_lrd)
2349                 tp->t_flags |= TF_LRD;
2350
2351         return (tp);
2352 }
2353
2354 /*
2355  * Drop a TCP connection, reporting
2356  * the specified error.  If connection is synchronized,
2357  * then send a RST to peer.
2358  */
2359 struct tcpcb *
2360 tcp_drop(struct tcpcb *tp, int errno)
2361 {
2362         struct socket *so = tptosocket(tp);
2363
2364         NET_EPOCH_ASSERT();
2365         INP_WLOCK_ASSERT(tptoinpcb(tp));
2366
2367         if (TCPS_HAVERCVDSYN(tp->t_state)) {
2368                 tcp_state_change(tp, TCPS_CLOSED);
2369                 /* Don't use tcp_output() here due to possible recursion. */
2370                 (void)tcp_output_nodrop(tp);
2371                 TCPSTAT_INC(tcps_drops);
2372         } else
2373                 TCPSTAT_INC(tcps_conndrops);
2374         if (errno == ETIMEDOUT && tp->t_softerror)
2375                 errno = tp->t_softerror;
2376         so->so_error = errno;
2377         return (tcp_close(tp));
2378 }
2379
2380 void
2381 tcp_discardcb(struct tcpcb *tp)
2382 {
2383         struct inpcb *inp = tptoinpcb(tp);
2384         struct socket *so = tptosocket(tp);
2385         struct mbuf *m;
2386 #ifdef INET6
2387         bool isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
2388 #endif
2389
2390         INP_WLOCK_ASSERT(inp);
2391
2392         tcp_timer_stop(tp);
2393         if (tp->t_fb->tfb_tcp_timer_stop_all) {
2394                 tp->t_fb->tfb_tcp_timer_stop_all(tp);
2395         }
2396
2397         /* free the reassembly queue, if any */
2398         tcp_reass_flush(tp);
2399
2400 #ifdef TCP_OFFLOAD
2401         /* Disconnect offload device, if any. */
2402         if (tp->t_flags & TF_TOE)
2403                 tcp_offload_detach(tp);
2404 #endif
2405
2406         tcp_free_sackholes(tp);
2407
2408 #ifdef TCPPCAP
2409         /* Free the TCP PCAP queues. */
2410         tcp_pcap_drain(&(tp->t_inpkts));
2411         tcp_pcap_drain(&(tp->t_outpkts));
2412 #endif
2413
2414         /* Allow the CC algorithm to clean up after itself. */
2415         if (CC_ALGO(tp)->cb_destroy != NULL)
2416                 CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
2417         CC_DATA(tp) = NULL;
2418         /* Detach from the CC algorithm */
2419         cc_detach(tp);
2420
2421 #ifdef TCP_HHOOK
2422         khelp_destroy_osd(&tp->t_osd);
2423 #endif
2424 #ifdef STATS
2425         stats_blob_destroy(tp->t_stats);
2426 #endif
2427
2428         CC_ALGO(tp) = NULL;
2429         if ((m = STAILQ_FIRST(&tp->t_inqueue)) != NULL) {
2430                 struct mbuf *prev;
2431
2432                 STAILQ_INIT(&tp->t_inqueue);
2433                 STAILQ_FOREACH_FROM_SAFE(m, &tp->t_inqueue, m_stailqpkt, prev)
2434                         m_freem(m);
2435         }
2436         TCPSTATES_DEC(tp->t_state);
2437
2438         if (tp->t_fb->tfb_tcp_fb_fini)
2439                 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2440         MPASS(!tcp_in_hpts(tp));
2441 #ifdef TCP_BLACKBOX
2442         tcp_log_tcpcbfini(tp);
2443 #endif
2444
2445         /*
2446          * If we got enough samples through the srtt filter,
2447          * save the rtt and rttvar in the routing entry.
2448          * 'Enough' is arbitrarily defined as 4 rtt samples.
2449          * 4 samples is enough for the srtt filter to converge
2450          * to within enough % of the correct value; fewer samples
2451          * and we could save a bogus rtt. The danger is not high
2452          * as tcp quickly recovers from everything.
2453          * XXX: Works very well but needs some more statistics!
2454          *
2455          * XXXRRS: Updating must be after the stack fini() since
2456          * that may be converting some internal representation of
2457          * say srtt etc into the general one used by other stacks.
2458          * Lets also at least protect against the so being NULL
2459          * as RW stated below.
2460          */
2461         if ((tp->t_rttupdated >= 4) && (so != NULL)) {
2462                 struct hc_metrics_lite metrics;
2463                 uint32_t ssthresh;
2464
2465                 bzero(&metrics, sizeof(metrics));
2466                 /*
2467                  * Update the ssthresh always when the conditions below
2468                  * are satisfied. This gives us better new start value
2469                  * for the congestion avoidance for new connections.
2470                  * ssthresh is only set if packet loss occurred on a session.
2471                  *
2472                  * XXXRW: 'so' may be NULL here, and/or socket buffer may be
2473                  * being torn down.  Ideally this code would not use 'so'.
2474                  */
2475                 ssthresh = tp->snd_ssthresh;
2476                 if (ssthresh != 0 && ssthresh < so->so_snd.sb_hiwat / 2) {
2477                         /*
2478                          * convert the limit from user data bytes to
2479                          * packets then to packet data bytes.
2480                          */
2481                         ssthresh = (ssthresh + tp->t_maxseg / 2) / tp->t_maxseg;
2482                         if (ssthresh < 2)
2483                                 ssthresh = 2;
2484                         ssthresh *= (tp->t_maxseg +
2485 #ifdef INET6
2486                             (isipv6 ? sizeof (struct ip6_hdr) +
2487                             sizeof (struct tcphdr) :
2488 #endif
2489                             sizeof (struct tcpiphdr)
2490 #ifdef INET6
2491                             )
2492 #endif
2493                             );
2494                 } else
2495                         ssthresh = 0;
2496                 metrics.rmx_ssthresh = ssthresh;
2497
2498                 metrics.rmx_rtt = tp->t_srtt;
2499                 metrics.rmx_rttvar = tp->t_rttvar;
2500                 metrics.rmx_cwnd = tp->snd_cwnd;
2501                 metrics.rmx_sendpipe = 0;
2502                 metrics.rmx_recvpipe = 0;
2503
2504                 tcp_hc_update(&inp->inp_inc, &metrics);
2505         }
2506
2507         refcount_release(&tp->t_fb->tfb_refcnt);
2508 }
2509
2510 /*
2511  * Attempt to close a TCP control block, marking it as dropped, and freeing
2512  * the socket if we hold the only reference.
2513  */
2514 struct tcpcb *
2515 tcp_close(struct tcpcb *tp)
2516 {
2517         struct inpcb *inp = tptoinpcb(tp);
2518         struct socket *so = tptosocket(tp);
2519
2520         INP_WLOCK_ASSERT(inp);
2521
2522 #ifdef TCP_OFFLOAD
2523         if (tp->t_state == TCPS_LISTEN)
2524                 tcp_offload_listen_stop(tp);
2525 #endif
2526         /*
2527          * This releases the TFO pending counter resource for TFO listen
2528          * sockets as well as passively-created TFO sockets that transition
2529          * from SYN_RECEIVED to CLOSED.
2530          */
2531         if (tp->t_tfo_pending) {
2532                 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
2533                 tp->t_tfo_pending = NULL;
2534         }
2535 #ifdef TCPHPTS
2536         tcp_hpts_remove(tp);
2537 #endif
2538         in_pcbdrop(inp);
2539         TCPSTAT_INC(tcps_closed);
2540         if (tp->t_state != TCPS_CLOSED)
2541                 tcp_state_change(tp, TCPS_CLOSED);
2542         KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
2543         soisdisconnected(so);
2544         if (inp->inp_flags & INP_SOCKREF) {
2545                 inp->inp_flags &= ~INP_SOCKREF;
2546                 INP_WUNLOCK(inp);
2547                 sorele(so);
2548                 return (NULL);
2549         }
2550         return (tp);
2551 }
2552
2553 /*
2554  * Notify a tcp user of an asynchronous error;
2555  * store error as soft error, but wake up user
2556  * (for now, won't do anything until can select for soft error).
2557  *
2558  * Do not wake up user since there currently is no mechanism for
2559  * reporting soft errors (yet - a kqueue filter may be added).
2560  */
2561 static struct inpcb *
2562 tcp_notify(struct inpcb *inp, int error)
2563 {
2564         struct tcpcb *tp;
2565
2566         INP_WLOCK_ASSERT(inp);
2567
2568         tp = intotcpcb(inp);
2569         KASSERT(tp != NULL, ("tcp_notify: tp == NULL"));
2570
2571         /*
2572          * Ignore some errors if we are hooked up.
2573          * If connection hasn't completed, has retransmitted several times,
2574          * and receives a second error, give up now.  This is better
2575          * than waiting a long time to establish a connection that
2576          * can never complete.
2577          */
2578         if (tp->t_state == TCPS_ESTABLISHED &&
2579             (error == EHOSTUNREACH || error == ENETUNREACH ||
2580              error == EHOSTDOWN)) {
2581                 if (inp->inp_route.ro_nh) {
2582                         NH_FREE(inp->inp_route.ro_nh);
2583                         inp->inp_route.ro_nh = (struct nhop_object *)NULL;
2584                 }
2585                 return (inp);
2586         } else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 &&
2587             tp->t_softerror) {
2588                 tp = tcp_drop(tp, error);
2589                 if (tp != NULL)
2590                         return (inp);
2591                 else
2592                         return (NULL);
2593         } else {
2594                 tp->t_softerror = error;
2595                 return (inp);
2596         }
2597 #if 0
2598         wakeup( &so->so_timeo);
2599         sorwakeup(so);
2600         sowwakeup(so);
2601 #endif
2602 }
2603
2604 static int
2605 tcp_pcblist(SYSCTL_HANDLER_ARGS)
2606 {
2607         struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
2608             INPLOOKUP_RLOCKPCB);
2609         struct xinpgen xig;
2610         struct inpcb *inp;
2611         int error;
2612
2613         if (req->newptr != NULL)
2614                 return (EPERM);
2615
2616         if (req->oldptr == NULL) {
2617                 int n;
2618
2619                 n = V_tcbinfo.ipi_count +
2620                     counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2621                 n += imax(n / 8, 10);
2622                 req->oldidx = 2 * (sizeof xig) + n * sizeof(struct xtcpcb);
2623                 return (0);
2624         }
2625
2626         if ((error = sysctl_wire_old_buffer(req, 0)) != 0)
2627                 return (error);
2628
2629         bzero(&xig, sizeof(xig));
2630         xig.xig_len = sizeof xig;
2631         xig.xig_count = V_tcbinfo.ipi_count +
2632             counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2633         xig.xig_gen = V_tcbinfo.ipi_gencnt;
2634         xig.xig_sogen = so_gencnt;
2635         error = SYSCTL_OUT(req, &xig, sizeof xig);
2636         if (error)
2637                 return (error);
2638
2639         error = syncache_pcblist(req);
2640         if (error)
2641                 return (error);
2642
2643         while ((inp = inp_next(&inpi)) != NULL) {
2644                 if (inp->inp_gencnt <= xig.xig_gen &&
2645                     cr_canseeinpcb(req->td->td_ucred, inp) == 0) {
2646                         struct xtcpcb xt;
2647
2648                         tcp_inptoxtp(inp, &xt);
2649                         error = SYSCTL_OUT(req, &xt, sizeof xt);
2650                         if (error) {
2651                                 INP_RUNLOCK(inp);
2652                                 break;
2653                         } else
2654                                 continue;
2655                 }
2656         }
2657
2658         if (!error) {
2659                 /*
2660                  * Give the user an updated idea of our state.
2661                  * If the generation differs from what we told
2662                  * her before, she knows that something happened
2663                  * while we were processing this request, and it
2664                  * might be necessary to retry.
2665                  */
2666                 xig.xig_gen = V_tcbinfo.ipi_gencnt;
2667                 xig.xig_sogen = so_gencnt;
2668                 xig.xig_count = V_tcbinfo.ipi_count +
2669                     counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2670                 error = SYSCTL_OUT(req, &xig, sizeof xig);
2671         }
2672
2673         return (error);
2674 }
2675
2676 SYSCTL_PROC(_net_inet_tcp, TCPCTL_PCBLIST, pcblist,
2677     CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
2678     NULL, 0, tcp_pcblist, "S,xtcpcb",
2679     "List of active TCP connections");
2680
2681 #ifdef INET
2682 static int
2683 tcp_getcred(SYSCTL_HANDLER_ARGS)
2684 {
2685         struct xucred xuc;
2686         struct sockaddr_in addrs[2];
2687         struct epoch_tracker et;
2688         struct inpcb *inp;
2689         int error;
2690
2691         error = priv_check(req->td, PRIV_NETINET_GETCRED);
2692         if (error)
2693                 return (error);
2694         error = SYSCTL_IN(req, addrs, sizeof(addrs));
2695         if (error)
2696                 return (error);
2697         NET_EPOCH_ENTER(et);
2698         inp = in_pcblookup(&V_tcbinfo, addrs[1].sin_addr, addrs[1].sin_port,
2699             addrs[0].sin_addr, addrs[0].sin_port, INPLOOKUP_RLOCKPCB, NULL);
2700         NET_EPOCH_EXIT(et);
2701         if (inp != NULL) {
2702                 if (error == 0)
2703                         error = cr_canseeinpcb(req->td->td_ucred, inp);
2704                 if (error == 0)
2705                         cru2x(inp->inp_cred, &xuc);
2706                 INP_RUNLOCK(inp);
2707         } else
2708                 error = ENOENT;
2709         if (error == 0)
2710                 error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
2711         return (error);
2712 }
2713
2714 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, getcred,
2715     CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_NEEDGIANT,
2716     0, 0, tcp_getcred, "S,xucred",
2717     "Get the xucred of a TCP connection");
2718 #endif /* INET */
2719
2720 #ifdef INET6
2721 static int
2722 tcp6_getcred(SYSCTL_HANDLER_ARGS)
2723 {
2724         struct epoch_tracker et;
2725         struct xucred xuc;
2726         struct sockaddr_in6 addrs[2];
2727         struct inpcb *inp;
2728         int error;
2729 #ifdef INET
2730         int mapped = 0;
2731 #endif
2732
2733         error = priv_check(req->td, PRIV_NETINET_GETCRED);
2734         if (error)
2735                 return (error);
2736         error = SYSCTL_IN(req, addrs, sizeof(addrs));
2737         if (error)
2738                 return (error);
2739         if ((error = sa6_embedscope(&addrs[0], V_ip6_use_defzone)) != 0 ||
2740             (error = sa6_embedscope(&addrs[1], V_ip6_use_defzone)) != 0) {
2741                 return (error);
2742         }
2743         if (IN6_IS_ADDR_V4MAPPED(&addrs[0].sin6_addr)) {
2744 #ifdef INET
2745                 if (IN6_IS_ADDR_V4MAPPED(&addrs[1].sin6_addr))
2746                         mapped = 1;
2747                 else
2748 #endif
2749                         return (EINVAL);
2750         }
2751
2752         NET_EPOCH_ENTER(et);
2753 #ifdef INET
2754         if (mapped == 1)
2755                 inp = in_pcblookup(&V_tcbinfo,
2756                         *(struct in_addr *)&addrs[1].sin6_addr.s6_addr[12],
2757                         addrs[1].sin6_port,
2758                         *(struct in_addr *)&addrs[0].sin6_addr.s6_addr[12],
2759                         addrs[0].sin6_port, INPLOOKUP_RLOCKPCB, NULL);
2760         else
2761 #endif
2762                 inp = in6_pcblookup(&V_tcbinfo,
2763                         &addrs[1].sin6_addr, addrs[1].sin6_port,
2764                         &addrs[0].sin6_addr, addrs[0].sin6_port,
2765                         INPLOOKUP_RLOCKPCB, NULL);
2766         NET_EPOCH_EXIT(et);
2767         if (inp != NULL) {
2768                 if (error == 0)
2769                         error = cr_canseeinpcb(req->td->td_ucred, inp);
2770                 if (error == 0)
2771                         cru2x(inp->inp_cred, &xuc);
2772                 INP_RUNLOCK(inp);
2773         } else
2774                 error = ENOENT;
2775         if (error == 0)
2776                 error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
2777         return (error);
2778 }
2779
2780 SYSCTL_PROC(_net_inet6_tcp6, OID_AUTO, getcred,
2781     CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_NEEDGIANT,
2782     0, 0, tcp6_getcred, "S,xucred",
2783     "Get the xucred of a TCP6 connection");
2784 #endif /* INET6 */
2785
2786 #ifdef INET
2787 /* Path MTU to try next when a fragmentation-needed message is received. */
2788 static inline int
2789 tcp_next_pmtu(const struct icmp *icp, const struct ip *ip)
2790 {
2791         int mtu = ntohs(icp->icmp_nextmtu);
2792
2793         /* If no alternative MTU was proposed, try the next smaller one. */
2794         if (!mtu)
2795                 mtu = ip_next_mtu(ntohs(ip->ip_len), 1);
2796         if (mtu < V_tcp_minmss + sizeof(struct tcpiphdr))
2797                 mtu = V_tcp_minmss + sizeof(struct tcpiphdr);
2798
2799         return (mtu);
2800 }
2801
2802 static void
2803 tcp_ctlinput_with_port(struct icmp *icp, uint16_t port)
2804 {
2805         struct ip *ip;
2806         struct tcphdr *th;
2807         struct inpcb *inp;
2808         struct tcpcb *tp;
2809         struct inpcb *(*notify)(struct inpcb *, int);
2810         struct in_conninfo inc;
2811         tcp_seq icmp_tcp_seq;
2812         int errno, mtu;
2813
2814         errno = icmp_errmap(icp);
2815         switch (errno) {
2816         case 0:
2817                 return;
2818         case EMSGSIZE:
2819                 notify = tcp_mtudisc_notify;
2820                 break;
2821         case ECONNREFUSED:
2822                 if (V_icmp_may_rst)
2823                         notify = tcp_drop_syn_sent;
2824                 else
2825                         notify = tcp_notify;
2826                 break;
2827         case EHOSTUNREACH:
2828                 if (V_icmp_may_rst && icp->icmp_type == ICMP_TIMXCEED)
2829                         notify = tcp_drop_syn_sent;
2830                 else
2831                         notify = tcp_notify;
2832                 break;
2833         default:
2834                 notify = tcp_notify;
2835         }
2836
2837         ip = &icp->icmp_ip;
2838         th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2839         icmp_tcp_seq = th->th_seq;
2840         inp = in_pcblookup(&V_tcbinfo, ip->ip_dst, th->th_dport, ip->ip_src,
2841             th->th_sport, INPLOOKUP_WLOCKPCB, NULL);
2842         if (inp != NULL)  {
2843                 tp = intotcpcb(inp);
2844 #ifdef TCP_OFFLOAD
2845                 if (tp->t_flags & TF_TOE && errno == EMSGSIZE) {
2846                         /*
2847                          * MTU discovery for offloaded connections.  Let
2848                          * the TOE driver verify seq# and process it.
2849                          */
2850                         mtu = tcp_next_pmtu(icp, ip);
2851                         tcp_offload_pmtu_update(tp, icmp_tcp_seq, mtu);
2852                         goto out;
2853                 }
2854 #endif
2855                 if (tp->t_port != port)
2856                         goto out;
2857                 if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
2858                     SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
2859                         if (errno == EMSGSIZE) {
2860                                 /*
2861                                  * MTU discovery: we got a needfrag and
2862                                  * will potentially try a lower MTU.
2863                                  */
2864                                 mtu = tcp_next_pmtu(icp, ip);
2865
2866                                 /*
2867                                  * Only process the offered MTU if it
2868                                  * is smaller than the current one.
2869                                  */
2870                                 if (mtu < tp->t_maxseg +
2871                                     sizeof(struct tcpiphdr)) {
2872                                         bzero(&inc, sizeof(inc));
2873                                         inc.inc_faddr = ip->ip_dst;
2874                                         inc.inc_fibnum =
2875                                             inp->inp_inc.inc_fibnum;
2876                                         tcp_hc_updatemtu(&inc, mtu);
2877                                         inp = tcp_mtudisc(inp, mtu);
2878                                 }
2879                         } else
2880                                 inp = (*notify)(inp, errno);
2881                 }
2882         } else {
2883                 bzero(&inc, sizeof(inc));
2884                 inc.inc_fport = th->th_dport;
2885                 inc.inc_lport = th->th_sport;
2886                 inc.inc_faddr = ip->ip_dst;
2887                 inc.inc_laddr = ip->ip_src;
2888                 syncache_unreach(&inc, icmp_tcp_seq, port);
2889         }
2890 out:
2891         if (inp != NULL)
2892                 INP_WUNLOCK(inp);
2893 }
2894
2895 static void
2896 tcp_ctlinput(struct icmp *icmp)
2897 {
2898         tcp_ctlinput_with_port(icmp, htons(0));
2899 }
2900
2901 static void
2902 tcp_ctlinput_viaudp(udp_tun_icmp_param_t param)
2903 {
2904         /* Its a tunneled TCP over UDP icmp */
2905         struct icmp *icmp = param.icmp;
2906         struct ip *outer_ip, *inner_ip;
2907         struct udphdr *udp;
2908         struct tcphdr *th, ttemp;
2909         int i_hlen, o_len;
2910         uint16_t port;
2911
2912         outer_ip = (struct ip *)((caddr_t)icmp - sizeof(struct ip));
2913         inner_ip = &icmp->icmp_ip;
2914         i_hlen = inner_ip->ip_hl << 2;
2915         o_len = ntohs(outer_ip->ip_len);
2916         if (o_len <
2917             (sizeof(struct ip) + 8 + i_hlen + sizeof(struct udphdr) + offsetof(struct tcphdr, th_ack))) {
2918                 /* Not enough data present */
2919                 return;
2920         }
2921         /* Ok lets strip out the inner udphdr header by copying up on top of it the tcp hdr */
2922         udp = (struct udphdr *)(((caddr_t)inner_ip) + i_hlen);
2923         if (ntohs(udp->uh_sport) != V_tcp_udp_tunneling_port) {
2924                 return;
2925         }
2926         port = udp->uh_dport;
2927         th = (struct tcphdr *)(udp + 1);
2928         memcpy(&ttemp, th, sizeof(struct tcphdr));
2929         memcpy(udp, &ttemp, sizeof(struct tcphdr));
2930         /* Now adjust down the size of the outer IP header */
2931         o_len -= sizeof(struct udphdr);
2932         outer_ip->ip_len = htons(o_len);
2933         /* Now call in to the normal handling code */
2934         tcp_ctlinput_with_port(icmp, port);
2935 }
2936 #endif /* INET */
2937
2938 #ifdef INET6
2939 static inline int
2940 tcp6_next_pmtu(const struct icmp6_hdr *icmp6)
2941 {
2942         int mtu = ntohl(icmp6->icmp6_mtu);
2943
2944         /*
2945          * If no alternative MTU was proposed, or the proposed MTU was too
2946          * small, set to the min.
2947          */
2948         if (mtu < IPV6_MMTU)
2949                 mtu = IPV6_MMTU - 8;    /* XXXNP: what is the adjustment for? */
2950         return (mtu);
2951 }
2952
2953 static void
2954 tcp6_ctlinput_with_port(struct ip6ctlparam *ip6cp, uint16_t port)
2955 {
2956         struct in6_addr *dst;
2957         struct inpcb *(*notify)(struct inpcb *, int);
2958         struct ip6_hdr *ip6;
2959         struct mbuf *m;
2960         struct inpcb *inp;
2961         struct tcpcb *tp;
2962         struct icmp6_hdr *icmp6;
2963         struct in_conninfo inc;
2964         struct tcp_ports {
2965                 uint16_t th_sport;
2966                 uint16_t th_dport;
2967         } t_ports;
2968         tcp_seq icmp_tcp_seq;
2969         unsigned int mtu;
2970         unsigned int off;
2971         int errno;
2972
2973         icmp6 = ip6cp->ip6c_icmp6;
2974         m = ip6cp->ip6c_m;
2975         ip6 = ip6cp->ip6c_ip6;
2976         off = ip6cp->ip6c_off;
2977         dst = &ip6cp->ip6c_finaldst->sin6_addr;
2978
2979         errno = icmp6_errmap(icmp6);
2980         switch (errno) {
2981         case 0:
2982                 return;
2983         case EMSGSIZE:
2984                 notify = tcp_mtudisc_notify;
2985                 break;
2986         case ECONNREFUSED:
2987                 if (V_icmp_may_rst)
2988                         notify = tcp_drop_syn_sent;
2989                 else
2990                         notify = tcp_notify;
2991                 break;
2992         case EHOSTUNREACH:
2993                 /*
2994                  * There are only four ICMPs that may reset connection:
2995                  * - administratively prohibited
2996                  * - port unreachable
2997                  * - time exceeded in transit
2998                  * - unknown next header
2999                  */
3000                 if (V_icmp_may_rst &&
3001                     ((icmp6->icmp6_type == ICMP6_DST_UNREACH &&
3002                      (icmp6->icmp6_code == ICMP6_DST_UNREACH_ADMIN ||
3003                       icmp6->icmp6_code == ICMP6_DST_UNREACH_NOPORT)) ||
3004                     (icmp6->icmp6_type == ICMP6_TIME_EXCEEDED &&
3005                       icmp6->icmp6_code == ICMP6_TIME_EXCEED_TRANSIT) ||
3006                     (icmp6->icmp6_type == ICMP6_PARAM_PROB &&
3007                       icmp6->icmp6_code == ICMP6_PARAMPROB_NEXTHEADER)))
3008                         notify = tcp_drop_syn_sent;
3009                 else
3010                         notify = tcp_notify;
3011                 break;
3012         default:
3013                 notify = tcp_notify;
3014         }
3015
3016         /* Check if we can safely get the ports from the tcp hdr */
3017         if (m == NULL ||
3018             (m->m_pkthdr.len <
3019                 (int32_t) (off + sizeof(struct tcp_ports)))) {
3020                 return;
3021         }
3022         bzero(&t_ports, sizeof(struct tcp_ports));
3023         m_copydata(m, off, sizeof(struct tcp_ports), (caddr_t)&t_ports);
3024         inp = in6_pcblookup(&V_tcbinfo, &ip6->ip6_dst, t_ports.th_dport,
3025             &ip6->ip6_src, t_ports.th_sport, INPLOOKUP_WLOCKPCB, NULL);
3026         off += sizeof(struct tcp_ports);
3027         if (m->m_pkthdr.len < (int32_t) (off + sizeof(tcp_seq))) {
3028                 goto out;
3029         }
3030         m_copydata(m, off, sizeof(tcp_seq), (caddr_t)&icmp_tcp_seq);
3031         if (inp != NULL)  {
3032                 tp = intotcpcb(inp);
3033 #ifdef TCP_OFFLOAD
3034                 if (tp->t_flags & TF_TOE && errno == EMSGSIZE) {
3035                         /* MTU discovery for offloaded connections. */
3036                         mtu = tcp6_next_pmtu(icmp6);
3037                         tcp_offload_pmtu_update(tp, icmp_tcp_seq, mtu);
3038                         goto out;
3039                 }
3040 #endif
3041                 if (tp->t_port != port)
3042                         goto out;
3043                 if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
3044                     SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
3045                         if (errno == EMSGSIZE) {
3046                                 /*
3047                                  * MTU discovery:
3048                                  * If we got a needfrag set the MTU
3049                                  * in the route to the suggested new
3050                                  * value (if given) and then notify.
3051                                  */
3052                                 mtu = tcp6_next_pmtu(icmp6);
3053
3054                                 bzero(&inc, sizeof(inc));
3055                                 inc.inc_fibnum = M_GETFIB(m);
3056                                 inc.inc_flags |= INC_ISIPV6;
3057                                 inc.inc6_faddr = *dst;
3058                                 if (in6_setscope(&inc.inc6_faddr,
3059                                         m->m_pkthdr.rcvif, NULL))
3060                                         goto out;
3061                                 /*
3062                                  * Only process the offered MTU if it
3063                                  * is smaller than the current one.
3064                                  */
3065                                 if (mtu < tp->t_maxseg +
3066                                     sizeof (struct tcphdr) +
3067                                     sizeof (struct ip6_hdr)) {
3068                                         tcp_hc_updatemtu(&inc, mtu);
3069                                         tcp_mtudisc(inp, mtu);
3070                                         ICMP6STAT_INC(icp6s_pmtuchg);
3071                                 }
3072                         } else
3073                                 inp = (*notify)(inp, errno);
3074                 }
3075         } else {
3076                 bzero(&inc, sizeof(inc));
3077                 inc.inc_fibnum = M_GETFIB(m);
3078                 inc.inc_flags |= INC_ISIPV6;
3079                 inc.inc_fport = t_ports.th_dport;
3080                 inc.inc_lport = t_ports.th_sport;
3081                 inc.inc6_faddr = *dst;
3082                 inc.inc6_laddr = ip6->ip6_src;
3083                 syncache_unreach(&inc, icmp_tcp_seq, port);
3084         }
3085 out:
3086         if (inp != NULL)
3087                 INP_WUNLOCK(inp);
3088 }
3089
3090 static void
3091 tcp6_ctlinput(struct ip6ctlparam *ctl)
3092 {
3093         tcp6_ctlinput_with_port(ctl, htons(0));
3094 }
3095
3096 static void
3097 tcp6_ctlinput_viaudp(udp_tun_icmp_param_t param)
3098 {
3099         struct ip6ctlparam *ip6cp = param.ip6cp;
3100         struct mbuf *m;
3101         struct udphdr *udp;
3102         uint16_t port;
3103
3104         m = m_pulldown(ip6cp->ip6c_m, ip6cp->ip6c_off, sizeof(struct udphdr), NULL);
3105         if (m == NULL) {
3106                 return;
3107         }
3108         udp = mtod(m, struct udphdr *);
3109         if (ntohs(udp->uh_sport) != V_tcp_udp_tunneling_port) {
3110                 return;
3111         }
3112         port = udp->uh_dport;
3113         m_adj(m, sizeof(struct udphdr));
3114         if ((m->m_flags & M_PKTHDR) == 0) {
3115                 ip6cp->ip6c_m->m_pkthdr.len -= sizeof(struct udphdr);
3116         }
3117         /* Now call in to the normal handling code */
3118         tcp6_ctlinput_with_port(ip6cp, port);
3119 }
3120
3121 #endif /* INET6 */
3122
3123 static uint32_t
3124 tcp_keyed_hash(struct in_conninfo *inc, u_char *key, u_int len)
3125 {
3126         SIPHASH_CTX ctx;
3127         uint32_t hash[2];
3128
3129         KASSERT(len >= SIPHASH_KEY_LENGTH,
3130             ("%s: keylen %u too short ", __func__, len));
3131         SipHash24_Init(&ctx);
3132         SipHash_SetKey(&ctx, (uint8_t *)key);
3133         SipHash_Update(&ctx, &inc->inc_fport, sizeof(uint16_t));
3134         SipHash_Update(&ctx, &inc->inc_lport, sizeof(uint16_t));
3135         switch (inc->inc_flags & INC_ISIPV6) {
3136 #ifdef INET
3137         case 0:
3138                 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(struct in_addr));
3139                 SipHash_Update(&ctx, &inc->inc_laddr, sizeof(struct in_addr));
3140                 break;
3141 #endif
3142 #ifdef INET6
3143         case INC_ISIPV6:
3144                 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(struct in6_addr));
3145                 SipHash_Update(&ctx, &inc->inc6_laddr, sizeof(struct in6_addr));
3146                 break;
3147 #endif
3148         }
3149         SipHash_Final((uint8_t *)hash, &ctx);
3150
3151         return (hash[0] ^ hash[1]);
3152 }
3153
3154 uint32_t
3155 tcp_new_ts_offset(struct in_conninfo *inc)
3156 {
3157         struct in_conninfo inc_store, *local_inc;
3158
3159         if (!V_tcp_ts_offset_per_conn) {
3160                 memcpy(&inc_store, inc, sizeof(struct in_conninfo));
3161                 inc_store.inc_lport = 0;
3162                 inc_store.inc_fport = 0;
3163                 local_inc = &inc_store;
3164         } else {
3165                 local_inc = inc;
3166         }
3167         return (tcp_keyed_hash(local_inc, V_ts_offset_secret,
3168             sizeof(V_ts_offset_secret)));
3169 }
3170
3171 /*
3172  * Following is where TCP initial sequence number generation occurs.
3173  *
3174  * There are two places where we must use initial sequence numbers:
3175  * 1.  In SYN-ACK packets.
3176  * 2.  In SYN packets.
3177  *
3178  * All ISNs for SYN-ACK packets are generated by the syncache.  See
3179  * tcp_syncache.c for details.
3180  *
3181  * The ISNs in SYN packets must be monotonic; TIME_WAIT recycling
3182  * depends on this property.  In addition, these ISNs should be
3183  * unguessable so as to prevent connection hijacking.  To satisfy
3184  * the requirements of this situation, the algorithm outlined in
3185  * RFC 1948 is used, with only small modifications.
3186  *
3187  * Implementation details:
3188  *
3189  * Time is based off the system timer, and is corrected so that it
3190  * increases by one megabyte per second.  This allows for proper
3191  * recycling on high speed LANs while still leaving over an hour
3192  * before rollover.
3193  *
3194  * As reading the *exact* system time is too expensive to be done
3195  * whenever setting up a TCP connection, we increment the time
3196  * offset in two ways.  First, a small random positive increment
3197  * is added to isn_offset for each connection that is set up.
3198  * Second, the function tcp_isn_tick fires once per clock tick
3199  * and increments isn_offset as necessary so that sequence numbers
3200  * are incremented at approximately ISN_BYTES_PER_SECOND.  The
3201  * random positive increments serve only to ensure that the same
3202  * exact sequence number is never sent out twice (as could otherwise
3203  * happen when a port is recycled in less than the system tick
3204  * interval.)
3205  *
3206  * net.inet.tcp.isn_reseed_interval controls the number of seconds
3207  * between seeding of isn_secret.  This is normally set to zero,
3208  * as reseeding should not be necessary.
3209  *
3210  * Locking of the global variables isn_secret, isn_last_reseed, isn_offset,
3211  * isn_offset_old, and isn_ctx is performed using the ISN lock.  In
3212  * general, this means holding an exclusive (write) lock.
3213  */
3214
3215 #define ISN_BYTES_PER_SECOND 1048576
3216 #define ISN_STATIC_INCREMENT 4096
3217 #define ISN_RANDOM_INCREMENT (4096 - 1)
3218 #define ISN_SECRET_LENGTH    SIPHASH_KEY_LENGTH
3219
3220 VNET_DEFINE_STATIC(u_char, isn_secret[ISN_SECRET_LENGTH]);
3221 VNET_DEFINE_STATIC(int, isn_last);
3222 VNET_DEFINE_STATIC(int, isn_last_reseed);
3223 VNET_DEFINE_STATIC(u_int32_t, isn_offset);
3224 VNET_DEFINE_STATIC(u_int32_t, isn_offset_old);
3225
3226 #define V_isn_secret                    VNET(isn_secret)
3227 #define V_isn_last                      VNET(isn_last)
3228 #define V_isn_last_reseed               VNET(isn_last_reseed)
3229 #define V_isn_offset                    VNET(isn_offset)
3230 #define V_isn_offset_old                VNET(isn_offset_old)
3231
3232 tcp_seq
3233 tcp_new_isn(struct in_conninfo *inc)
3234 {
3235         tcp_seq new_isn;
3236         u_int32_t projected_offset;
3237
3238         ISN_LOCK();
3239         /* Seed if this is the first use, reseed if requested. */
3240         if ((V_isn_last_reseed == 0) || ((V_tcp_isn_reseed_interval > 0) &&
3241              (((u_int)V_isn_last_reseed + (u_int)V_tcp_isn_reseed_interval*hz)
3242                 < (u_int)ticks))) {
3243                 arc4rand(&V_isn_secret, sizeof(V_isn_secret), 0);
3244                 V_isn_last_reseed = ticks;
3245         }
3246
3247         /* Compute the hash and return the ISN. */
3248         new_isn = (tcp_seq)tcp_keyed_hash(inc, V_isn_secret,
3249             sizeof(V_isn_secret));
3250         V_isn_offset += ISN_STATIC_INCREMENT +
3251                 (arc4random() & ISN_RANDOM_INCREMENT);
3252         if (ticks != V_isn_last) {
3253                 projected_offset = V_isn_offset_old +
3254                     ISN_BYTES_PER_SECOND / hz * (ticks - V_isn_last);
3255                 if (SEQ_GT(projected_offset, V_isn_offset))
3256                         V_isn_offset = projected_offset;
3257                 V_isn_offset_old = V_isn_offset;
3258                 V_isn_last = ticks;
3259         }
3260         new_isn += V_isn_offset;
3261         ISN_UNLOCK();
3262         return (new_isn);
3263 }
3264
3265 /*
3266  * When a specific ICMP unreachable message is received and the
3267  * connection state is SYN-SENT, drop the connection.  This behavior
3268  * is controlled by the icmp_may_rst sysctl.
3269  */
3270 static struct inpcb *
3271 tcp_drop_syn_sent(struct inpcb *inp, int errno)
3272 {
3273         struct tcpcb *tp;
3274
3275         NET_EPOCH_ASSERT();
3276         INP_WLOCK_ASSERT(inp);
3277
3278         tp = intotcpcb(inp);
3279         if (tp->t_state != TCPS_SYN_SENT)
3280                 return (inp);
3281
3282         if (IS_FASTOPEN(tp->t_flags))
3283                 tcp_fastopen_disable_path(tp);
3284
3285         tp = tcp_drop(tp, errno);
3286         if (tp != NULL)
3287                 return (inp);
3288         else
3289                 return (NULL);
3290 }
3291
3292 /*
3293  * When `need fragmentation' ICMP is received, update our idea of the MSS
3294  * based on the new value. Also nudge TCP to send something, since we
3295  * know the packet we just sent was dropped.
3296  * This duplicates some code in the tcp_mss() function in tcp_input.c.
3297  */
3298 static struct inpcb *
3299 tcp_mtudisc_notify(struct inpcb *inp, int error)
3300 {
3301
3302         return (tcp_mtudisc(inp, -1));
3303 }
3304
3305 static struct inpcb *
3306 tcp_mtudisc(struct inpcb *inp, int mtuoffer)
3307 {
3308         struct tcpcb *tp;
3309         struct socket *so;
3310
3311         INP_WLOCK_ASSERT(inp);
3312
3313         tp = intotcpcb(inp);
3314         KASSERT(tp != NULL, ("tcp_mtudisc: tp == NULL"));
3315
3316         tcp_mss_update(tp, -1, mtuoffer, NULL, NULL);
3317
3318         so = inp->inp_socket;
3319         SOCKBUF_LOCK(&so->so_snd);
3320         /* If the mss is larger than the socket buffer, decrease the mss. */
3321         if (so->so_snd.sb_hiwat < tp->t_maxseg)
3322                 tp->t_maxseg = so->so_snd.sb_hiwat;
3323         SOCKBUF_UNLOCK(&so->so_snd);
3324
3325         TCPSTAT_INC(tcps_mturesent);
3326         tp->t_rtttime = 0;
3327         tp->snd_nxt = tp->snd_una;
3328         tcp_free_sackholes(tp);
3329         tp->snd_recover = tp->snd_max;
3330         if (tp->t_flags & TF_SACK_PERMIT)
3331                 EXIT_FASTRECOVERY(tp->t_flags);
3332         if (tp->t_fb->tfb_tcp_mtu_chg != NULL) {
3333                 /*
3334                  * Conceptually the snd_nxt setting
3335                  * and freeing sack holes should
3336                  * be done by the default stacks
3337                  * own tfb_tcp_mtu_chg().
3338                  */
3339                 tp->t_fb->tfb_tcp_mtu_chg(tp);
3340         }
3341         if (tcp_output(tp) < 0)
3342                 return (NULL);
3343         else
3344                 return (inp);
3345 }
3346
3347 #ifdef INET
3348 /*
3349  * Look-up the routing entry to the peer of this inpcb.  If no route
3350  * is found and it cannot be allocated, then return 0.  This routine
3351  * is called by TCP routines that access the rmx structure and by
3352  * tcp_mss_update to get the peer/interface MTU.
3353  */
3354 uint32_t
3355 tcp_maxmtu(struct in_conninfo *inc, struct tcp_ifcap *cap)
3356 {
3357         struct nhop_object *nh;
3358         struct ifnet *ifp;
3359         uint32_t maxmtu = 0;
3360
3361         KASSERT(inc != NULL, ("tcp_maxmtu with NULL in_conninfo pointer"));
3362
3363         if (inc->inc_faddr.s_addr != INADDR_ANY) {
3364                 nh = fib4_lookup(inc->inc_fibnum, inc->inc_faddr, 0, NHR_NONE, 0);
3365                 if (nh == NULL)
3366                         return (0);
3367
3368                 ifp = nh->nh_ifp;
3369                 maxmtu = nh->nh_mtu;
3370
3371                 /* Report additional interface capabilities. */
3372                 if (cap != NULL) {
3373                         if (ifp->if_capenable & IFCAP_TSO4 &&
3374                             ifp->if_hwassist & CSUM_TSO) {
3375                                 cap->ifcap |= CSUM_TSO;
3376                                 cap->tsomax = ifp->if_hw_tsomax;
3377                                 cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
3378                                 cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
3379                         }
3380                 }
3381         }
3382         return (maxmtu);
3383 }
3384 #endif /* INET */
3385
3386 #ifdef INET6
3387 uint32_t
3388 tcp_maxmtu6(struct in_conninfo *inc, struct tcp_ifcap *cap)
3389 {
3390         struct nhop_object *nh;
3391         struct in6_addr dst6;
3392         uint32_t scopeid;
3393         struct ifnet *ifp;
3394         uint32_t maxmtu = 0;
3395
3396         KASSERT(inc != NULL, ("tcp_maxmtu6 with NULL in_conninfo pointer"));
3397
3398         if (inc->inc_flags & INC_IPV6MINMTU)
3399                 return (IPV6_MMTU);
3400
3401         if (!IN6_IS_ADDR_UNSPECIFIED(&inc->inc6_faddr)) {
3402                 in6_splitscope(&inc->inc6_faddr, &dst6, &scopeid);
3403                 nh = fib6_lookup(inc->inc_fibnum, &dst6, scopeid, NHR_NONE, 0);
3404                 if (nh == NULL)
3405                         return (0);
3406
3407                 ifp = nh->nh_ifp;
3408                 maxmtu = nh->nh_mtu;
3409
3410                 /* Report additional interface capabilities. */
3411                 if (cap != NULL) {
3412                         if (ifp->if_capenable & IFCAP_TSO6 &&
3413                             ifp->if_hwassist & CSUM_TSO) {
3414                                 cap->ifcap |= CSUM_TSO;
3415                                 cap->tsomax = ifp->if_hw_tsomax;
3416                                 cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
3417                                 cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
3418                         }
3419                 }
3420         }
3421
3422         return (maxmtu);
3423 }
3424
3425 /*
3426  * Handle setsockopt(IPV6_USE_MIN_MTU) by a TCP stack.
3427  *
3428  * XXXGL: we are updating inpcb here with INC_IPV6MINMTU flag.
3429  * The right place to do that is ip6_setpktopt() that has just been
3430  * executed.  By the way it just filled ip6po_minmtu for us.
3431  */
3432 void
3433 tcp6_use_min_mtu(struct tcpcb *tp)
3434 {
3435         struct inpcb *inp = tptoinpcb(tp);
3436
3437         INP_WLOCK_ASSERT(inp);
3438         /*
3439          * In case of the IPV6_USE_MIN_MTU socket
3440          * option, the INC_IPV6MINMTU flag to announce
3441          * a corresponding MSS during the initial
3442          * handshake.  If the TCP connection is not in
3443          * the front states, just reduce the MSS being
3444          * used.  This avoids the sending of TCP
3445          * segments which will be fragmented at the
3446          * IPv6 layer.
3447          */
3448         inp->inp_inc.inc_flags |= INC_IPV6MINMTU;
3449         if ((tp->t_state >= TCPS_SYN_SENT) &&
3450             (inp->inp_inc.inc_flags & INC_ISIPV6)) {
3451                 struct ip6_pktopts *opt;
3452
3453                 opt = inp->in6p_outputopts;
3454                 if (opt != NULL && opt->ip6po_minmtu == IP6PO_MINMTU_ALL &&
3455                     tp->t_maxseg > TCP6_MSS)
3456                         tp->t_maxseg = TCP6_MSS;
3457         }
3458 }
3459 #endif /* INET6 */
3460
3461 /*
3462  * Calculate effective SMSS per RFC5681 definition for a given TCP
3463  * connection at its current state, taking into account SACK and etc.
3464  */
3465 u_int
3466 tcp_maxseg(const struct tcpcb *tp)
3467 {
3468         u_int optlen;
3469
3470         if (tp->t_flags & TF_NOOPT)
3471                 return (tp->t_maxseg);
3472
3473         /*
3474          * Here we have a simplified code from tcp_addoptions(),
3475          * without a proper loop, and having most of paddings hardcoded.
3476          * We might make mistakes with padding here in some edge cases,
3477          * but this is harmless, since result of tcp_maxseg() is used
3478          * only in cwnd and ssthresh estimations.
3479          */
3480         if (TCPS_HAVEESTABLISHED(tp->t_state)) {
3481                 if (tp->t_flags & TF_RCVD_TSTMP)
3482                         optlen = TCPOLEN_TSTAMP_APPA;
3483                 else
3484                         optlen = 0;
3485 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3486                 if (tp->t_flags & TF_SIGNATURE)
3487                         optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3488 #endif
3489                 if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks > 0) {
3490                         optlen += TCPOLEN_SACKHDR;
3491                         optlen += tp->rcv_numsacks * TCPOLEN_SACK;
3492                         optlen = PADTCPOLEN(optlen);
3493                 }
3494         } else {
3495                 if (tp->t_flags & TF_REQ_TSTMP)
3496                         optlen = TCPOLEN_TSTAMP_APPA;
3497                 else
3498                         optlen = PADTCPOLEN(TCPOLEN_MAXSEG);
3499                 if (tp->t_flags & TF_REQ_SCALE)
3500                         optlen += PADTCPOLEN(TCPOLEN_WINDOW);
3501 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3502                 if (tp->t_flags & TF_SIGNATURE)
3503                         optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3504 #endif
3505                 if (tp->t_flags & TF_SACK_PERMIT)
3506                         optlen += PADTCPOLEN(TCPOLEN_SACK_PERMITTED);
3507         }
3508 #undef PAD
3509         optlen = min(optlen, TCP_MAXOLEN);
3510         return (tp->t_maxseg - optlen);
3511 }
3512
3513
3514 u_int
3515 tcp_fixed_maxseg(const struct tcpcb *tp)
3516 {
3517         int optlen;
3518
3519         if (tp->t_flags & TF_NOOPT)
3520                 return (tp->t_maxseg);
3521
3522         /*
3523          * Here we have a simplified code from tcp_addoptions(),
3524          * without a proper loop, and having most of paddings hardcoded.
3525          * We only consider fixed options that we would send every
3526          * time I.e. SACK is not considered. This is important
3527          * for cc modules to figure out what the modulo of the
3528          * cwnd should be.
3529          */
3530 #define PAD(len)        ((((len) / 4) + !!((len) % 4)) * 4)
3531         if (TCPS_HAVEESTABLISHED(tp->t_state)) {
3532                 if (tp->t_flags & TF_RCVD_TSTMP)
3533                         optlen = TCPOLEN_TSTAMP_APPA;
3534                 else
3535                         optlen = 0;
3536 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3537                 if (tp->t_flags & TF_SIGNATURE)
3538                         optlen += PAD(TCPOLEN_SIGNATURE);
3539 #endif
3540         } else {
3541                 if (tp->t_flags & TF_REQ_TSTMP)
3542                         optlen = TCPOLEN_TSTAMP_APPA;
3543                 else
3544                         optlen = PAD(TCPOLEN_MAXSEG);
3545                 if (tp->t_flags & TF_REQ_SCALE)
3546                         optlen += PAD(TCPOLEN_WINDOW);
3547 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3548                 if (tp->t_flags & TF_SIGNATURE)
3549                         optlen += PAD(TCPOLEN_SIGNATURE);
3550 #endif
3551                 if (tp->t_flags & TF_SACK_PERMIT)
3552                         optlen += PAD(TCPOLEN_SACK_PERMITTED);
3553         }
3554 #undef PAD
3555         optlen = min(optlen, TCP_MAXOLEN);
3556         return (tp->t_maxseg - optlen);
3557 }
3558
3559
3560
3561 static int
3562 sysctl_drop(SYSCTL_HANDLER_ARGS)
3563 {
3564         /* addrs[0] is a foreign socket, addrs[1] is a local one. */
3565         struct sockaddr_storage addrs[2];
3566         struct inpcb *inp;
3567         struct tcpcb *tp;
3568 #ifdef INET
3569         struct sockaddr_in *fin = NULL, *lin = NULL;
3570 #endif
3571         struct epoch_tracker et;
3572 #ifdef INET6
3573         struct sockaddr_in6 *fin6, *lin6;
3574 #endif
3575         int error;
3576
3577         inp = NULL;
3578 #ifdef INET6
3579         fin6 = lin6 = NULL;
3580 #endif
3581         error = 0;
3582
3583         if (req->oldptr != NULL || req->oldlen != 0)
3584                 return (EINVAL);
3585         if (req->newptr == NULL)
3586                 return (EPERM);
3587         if (req->newlen < sizeof(addrs))
3588                 return (ENOMEM);
3589         error = SYSCTL_IN(req, &addrs, sizeof(addrs));
3590         if (error)
3591                 return (error);
3592
3593         switch (addrs[0].ss_family) {
3594 #ifdef INET6
3595         case AF_INET6:
3596                 fin6 = (struct sockaddr_in6 *)&addrs[0];
3597                 lin6 = (struct sockaddr_in6 *)&addrs[1];
3598                 if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
3599                     lin6->sin6_len != sizeof(struct sockaddr_in6))
3600                         return (EINVAL);
3601                 if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
3602                         if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
3603                                 return (EINVAL);
3604                         in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
3605                         in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
3606 #ifdef INET
3607                         fin = (struct sockaddr_in *)&addrs[0];
3608                         lin = (struct sockaddr_in *)&addrs[1];
3609 #endif
3610                         break;
3611                 }
3612                 error = sa6_embedscope(fin6, V_ip6_use_defzone);
3613                 if (error)
3614                         return (error);
3615                 error = sa6_embedscope(lin6, V_ip6_use_defzone);
3616                 if (error)
3617                         return (error);
3618                 break;
3619 #endif
3620 #ifdef INET
3621         case AF_INET:
3622                 fin = (struct sockaddr_in *)&addrs[0];
3623                 lin = (struct sockaddr_in *)&addrs[1];
3624                 if (fin->sin_len != sizeof(struct sockaddr_in) ||
3625                     lin->sin_len != sizeof(struct sockaddr_in))
3626                         return (EINVAL);
3627                 break;
3628 #endif
3629         default:
3630                 return (EINVAL);
3631         }
3632         NET_EPOCH_ENTER(et);
3633         switch (addrs[0].ss_family) {
3634 #ifdef INET6
3635         case AF_INET6:
3636                 inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
3637                     fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
3638                     INPLOOKUP_WLOCKPCB, NULL);
3639                 break;
3640 #endif
3641 #ifdef INET
3642         case AF_INET:
3643                 inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
3644                     lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
3645                 break;
3646 #endif
3647         }
3648         if (inp != NULL) {
3649                 if (!SOLISTENING(inp->inp_socket)) {
3650                         tp = intotcpcb(inp);
3651                         tp = tcp_drop(tp, ECONNABORTED);
3652                         if (tp != NULL)
3653                                 INP_WUNLOCK(inp);
3654                 } else
3655                         INP_WUNLOCK(inp);
3656         } else
3657                 error = ESRCH;
3658         NET_EPOCH_EXIT(et);
3659         return (error);
3660 }
3661
3662 SYSCTL_PROC(_net_inet_tcp, TCPCTL_DROP, drop,
3663     CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3664     CTLFLAG_NEEDGIANT, NULL, 0, sysctl_drop, "",
3665     "Drop TCP connection");
3666
3667 static int
3668 tcp_sysctl_setsockopt(SYSCTL_HANDLER_ARGS)
3669 {
3670         return (sysctl_setsockopt(oidp, arg1, arg2, req, &V_tcbinfo,
3671             &tcp_ctloutput_set));
3672 }
3673
3674 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, setsockopt,
3675     CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3676     CTLFLAG_MPSAFE, NULL, 0, tcp_sysctl_setsockopt, "",
3677     "Set socket option for TCP endpoint");
3678
3679 #ifdef KERN_TLS
3680 static int
3681 sysctl_switch_tls(SYSCTL_HANDLER_ARGS)
3682 {
3683         /* addrs[0] is a foreign socket, addrs[1] is a local one. */
3684         struct sockaddr_storage addrs[2];
3685         struct inpcb *inp;
3686 #ifdef INET
3687         struct sockaddr_in *fin = NULL, *lin = NULL;
3688 #endif
3689         struct epoch_tracker et;
3690 #ifdef INET6
3691         struct sockaddr_in6 *fin6, *lin6;
3692 #endif
3693         int error;
3694
3695         inp = NULL;
3696 #ifdef INET6
3697         fin6 = lin6 = NULL;
3698 #endif
3699         error = 0;
3700
3701         if (req->oldptr != NULL || req->oldlen != 0)
3702                 return (EINVAL);
3703         if (req->newptr == NULL)
3704                 return (EPERM);
3705         if (req->newlen < sizeof(addrs))
3706                 return (ENOMEM);
3707         error = SYSCTL_IN(req, &addrs, sizeof(addrs));
3708         if (error)
3709                 return (error);
3710
3711         switch (addrs[0].ss_family) {
3712 #ifdef INET6
3713         case AF_INET6:
3714                 fin6 = (struct sockaddr_in6 *)&addrs[0];
3715                 lin6 = (struct sockaddr_in6 *)&addrs[1];
3716                 if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
3717                     lin6->sin6_len != sizeof(struct sockaddr_in6))
3718                         return (EINVAL);
3719                 if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
3720                         if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
3721                                 return (EINVAL);
3722                         in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
3723                         in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
3724 #ifdef INET
3725                         fin = (struct sockaddr_in *)&addrs[0];
3726                         lin = (struct sockaddr_in *)&addrs[1];
3727 #endif
3728                         break;
3729                 }
3730                 error = sa6_embedscope(fin6, V_ip6_use_defzone);
3731                 if (error)
3732                         return (error);
3733                 error = sa6_embedscope(lin6, V_ip6_use_defzone);
3734                 if (error)
3735                         return (error);
3736                 break;
3737 #endif
3738 #ifdef INET
3739         case AF_INET:
3740                 fin = (struct sockaddr_in *)&addrs[0];
3741                 lin = (struct sockaddr_in *)&addrs[1];
3742                 if (fin->sin_len != sizeof(struct sockaddr_in) ||
3743                     lin->sin_len != sizeof(struct sockaddr_in))
3744                         return (EINVAL);
3745                 break;
3746 #endif
3747         default:
3748                 return (EINVAL);
3749         }
3750         NET_EPOCH_ENTER(et);
3751         switch (addrs[0].ss_family) {
3752 #ifdef INET6
3753         case AF_INET6:
3754                 inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
3755                     fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
3756                     INPLOOKUP_WLOCKPCB, NULL);
3757                 break;
3758 #endif
3759 #ifdef INET
3760         case AF_INET:
3761                 inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
3762                     lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
3763                 break;
3764 #endif
3765         }
3766         NET_EPOCH_EXIT(et);
3767         if (inp != NULL) {
3768                 struct socket *so;
3769
3770                 so = inp->inp_socket;
3771                 soref(so);
3772                 error = ktls_set_tx_mode(so,
3773                     arg2 == 0 ? TCP_TLS_MODE_SW : TCP_TLS_MODE_IFNET);
3774                 INP_WUNLOCK(inp);
3775                 sorele(so);
3776         } else
3777                 error = ESRCH;
3778         return (error);
3779 }
3780
3781 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_sw_tls,
3782     CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3783     CTLFLAG_NEEDGIANT, NULL, 0, sysctl_switch_tls, "",
3784     "Switch TCP connection to SW TLS");
3785 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_ifnet_tls,
3786     CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3787     CTLFLAG_NEEDGIANT, NULL, 1, sysctl_switch_tls, "",
3788     "Switch TCP connection to ifnet TLS");
3789 #endif
3790
3791 /*
3792  * Generate a standardized TCP log line for use throughout the
3793  * tcp subsystem.  Memory allocation is done with M_NOWAIT to
3794  * allow use in the interrupt context.
3795  *
3796  * NB: The caller MUST free(s, M_TCPLOG) the returned string.
3797  * NB: The function may return NULL if memory allocation failed.
3798  *
3799  * Due to header inclusion and ordering limitations the struct ip
3800  * and ip6_hdr pointers have to be passed as void pointers.
3801  */
3802 char *
3803 tcp_log_vain(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3804     const void *ip6hdr)
3805 {
3806
3807         /* Is logging enabled? */
3808         if (V_tcp_log_in_vain == 0)
3809                 return (NULL);
3810
3811         return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
3812 }
3813
3814 char *
3815 tcp_log_addrs(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3816     const void *ip6hdr)
3817 {
3818
3819         /* Is logging enabled? */
3820         if (tcp_log_debug == 0)
3821                 return (NULL);
3822
3823         return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
3824 }
3825
3826 static char *
3827 tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3828     const void *ip6hdr)
3829 {
3830         char *s, *sp;
3831         size_t size;
3832 #ifdef INET
3833         const struct ip *ip = (const struct ip *)ip4hdr;
3834 #endif
3835 #ifdef INET6
3836         const struct ip6_hdr *ip6 = (const struct ip6_hdr *)ip6hdr;
3837 #endif /* INET6 */
3838
3839         /*
3840          * The log line looks like this:
3841          * "TCP: [1.2.3.4]:50332 to [1.2.3.4]:80 tcpflags 0x2<SYN>"
3842          */
3843         size = sizeof("TCP: []:12345 to []:12345 tcpflags 0x2<>") +
3844             sizeof(PRINT_TH_FLAGS) + 1 +
3845 #ifdef INET6
3846             2 * INET6_ADDRSTRLEN;
3847 #else
3848             2 * INET_ADDRSTRLEN;
3849 #endif /* INET6 */
3850
3851         s = malloc(size, M_TCPLOG, M_ZERO|M_NOWAIT);
3852         if (s == NULL)
3853                 return (NULL);
3854
3855         strcat(s, "TCP: [");
3856         sp = s + strlen(s);
3857
3858         if (inc && ((inc->inc_flags & INC_ISIPV6) == 0)) {
3859                 inet_ntoa_r(inc->inc_faddr, sp);
3860                 sp = s + strlen(s);
3861                 sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
3862                 sp = s + strlen(s);
3863                 inet_ntoa_r(inc->inc_laddr, sp);
3864                 sp = s + strlen(s);
3865                 sprintf(sp, "]:%i", ntohs(inc->inc_lport));
3866 #ifdef INET6
3867         } else if (inc) {
3868                 ip6_sprintf(sp, &inc->inc6_faddr);
3869                 sp = s + strlen(s);
3870                 sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
3871                 sp = s + strlen(s);
3872                 ip6_sprintf(sp, &inc->inc6_laddr);
3873                 sp = s + strlen(s);
3874                 sprintf(sp, "]:%i", ntohs(inc->inc_lport));
3875         } else if (ip6 && th) {
3876                 ip6_sprintf(sp, &ip6->ip6_src);
3877                 sp = s + strlen(s);
3878                 sprintf(sp, "]:%i to [", ntohs(th->th_sport));
3879                 sp = s + strlen(s);
3880                 ip6_sprintf(sp, &ip6->ip6_dst);
3881                 sp = s + strlen(s);
3882                 sprintf(sp, "]:%i", ntohs(th->th_dport));
3883 #endif /* INET6 */
3884 #ifdef INET
3885         } else if (ip && th) {
3886                 inet_ntoa_r(ip->ip_src, sp);
3887                 sp = s + strlen(s);
3888                 sprintf(sp, "]:%i to [", ntohs(th->th_sport));
3889                 sp = s + strlen(s);
3890                 inet_ntoa_r(ip->ip_dst, sp);
3891                 sp = s + strlen(s);
3892                 sprintf(sp, "]:%i", ntohs(th->th_dport));
3893 #endif /* INET */
3894         } else {
3895                 free(s, M_TCPLOG);
3896                 return (NULL);
3897         }
3898         sp = s + strlen(s);
3899         if (th)
3900                 sprintf(sp, " tcpflags 0x%b", tcp_get_flags(th), PRINT_TH_FLAGS);
3901         if (*(s + size - 1) != '\0')
3902                 panic("%s: string too long", __func__);
3903         return (s);
3904 }
3905
3906 /*
3907  * A subroutine which makes it easy to track TCP state changes with DTrace.
3908  * This function shouldn't be called for t_state initializations that don't
3909  * correspond to actual TCP state transitions.
3910  */
3911 void
3912 tcp_state_change(struct tcpcb *tp, int newstate)
3913 {
3914 #if defined(KDTRACE_HOOKS)
3915         int pstate = tp->t_state;
3916 #endif
3917
3918         TCPSTATES_DEC(tp->t_state);
3919         TCPSTATES_INC(newstate);
3920         tp->t_state = newstate;
3921         TCP_PROBE6(state__change, NULL, tp, NULL, tp, NULL, pstate);
3922 }
3923
3924 /*
3925  * Create an external-format (``xtcpcb'') structure using the information in
3926  * the kernel-format tcpcb structure pointed to by tp.  This is done to
3927  * reduce the spew of irrelevant information over this interface, to isolate
3928  * user code from changes in the kernel structure, and potentially to provide
3929  * information-hiding if we decide that some of this information should be
3930  * hidden from users.
3931  */
3932 void
3933 tcp_inptoxtp(const struct inpcb *inp, struct xtcpcb *xt)
3934 {
3935         struct tcpcb *tp = intotcpcb(inp);
3936         sbintime_t now;
3937
3938         bzero(xt, sizeof(*xt));
3939         xt->t_state = tp->t_state;
3940         xt->t_logstate = tcp_get_bblog_state(tp);
3941         xt->t_flags = tp->t_flags;
3942         xt->t_sndzerowin = tp->t_sndzerowin;
3943         xt->t_sndrexmitpack = tp->t_sndrexmitpack;
3944         xt->t_rcvoopack = tp->t_rcvoopack;
3945         xt->t_rcv_wnd = tp->rcv_wnd;
3946         xt->t_snd_wnd = tp->snd_wnd;
3947         xt->t_snd_cwnd = tp->snd_cwnd;
3948         xt->t_snd_ssthresh = tp->snd_ssthresh;
3949         xt->t_dsack_bytes = tp->t_dsack_bytes;
3950         xt->t_dsack_tlp_bytes = tp->t_dsack_tlp_bytes;
3951         xt->t_dsack_pack = tp->t_dsack_pack;
3952         xt->t_maxseg = tp->t_maxseg;
3953         xt->xt_ecn = (tp->t_flags2 & TF2_ECN_PERMIT) ? 1 : 0 +
3954                      (tp->t_flags2 & TF2_ACE_PERMIT) ? 2 : 0;
3955
3956         now = getsbinuptime();
3957 #define COPYTIMER(which,where)  do {                                    \
3958         if (tp->t_timers[which] != SBT_MAX)                             \
3959                 xt->where = (tp->t_timers[which] - now) / SBT_1MS;      \
3960         else                                                            \
3961                 xt->where = 0;                                          \
3962 } while (0)
3963         COPYTIMER(TT_DELACK, tt_delack);
3964         COPYTIMER(TT_REXMT, tt_rexmt);
3965         COPYTIMER(TT_PERSIST, tt_persist);
3966         COPYTIMER(TT_KEEP, tt_keep);
3967         COPYTIMER(TT_2MSL, tt_2msl);
3968 #undef COPYTIMER
3969         xt->t_rcvtime = 1000 * (ticks - tp->t_rcvtime) / hz;
3970
3971         xt->xt_encaps_port = tp->t_port;
3972         bcopy(tp->t_fb->tfb_tcp_block_name, xt->xt_stack,
3973             TCP_FUNCTION_NAME_LEN_MAX);
3974         bcopy(CC_ALGO(tp)->name, xt->xt_cc, TCP_CA_NAME_MAX);
3975 #ifdef TCP_BLACKBOX
3976         (void)tcp_log_get_id(tp, xt->xt_logid);
3977 #endif
3978
3979         xt->xt_len = sizeof(struct xtcpcb);
3980         in_pcbtoxinpcb(inp, &xt->xt_inp);
3981         /*
3982          * TCP doesn't use inp_ppcb pointer, we embed inpcb into tcpcb.
3983          * Fixup the pointer that in_pcbtoxinpcb() has set.  When printing
3984          * TCP netstat(1) used to use this pointer, so this fixup needs to
3985          * stay for stable/14.
3986          */
3987         xt->xt_inp.inp_ppcb = (uintptr_t)tp;
3988 }
3989
3990 void
3991 tcp_log_end_status(struct tcpcb *tp, uint8_t status)
3992 {
3993         uint32_t bit, i;
3994
3995         if ((tp == NULL) ||
3996             (status > TCP_EI_STATUS_MAX_VALUE) ||
3997             (status == 0)) {
3998                 /* Invalid */
3999                 return;
4000         }
4001         if (status > (sizeof(uint32_t) * 8)) {
4002                 /* Should this be a KASSERT? */
4003                 return;
4004         }
4005         bit = 1U << (status - 1);
4006         if (bit & tp->t_end_info_status) {
4007                 /* already logged */
4008                 return;
4009         }
4010         for (i = 0; i < TCP_END_BYTE_INFO; i++) {
4011                 if (tp->t_end_info_bytes[i] == TCP_EI_EMPTY_SLOT) {
4012                         tp->t_end_info_bytes[i] = status;
4013                         tp->t_end_info_status |= bit;
4014                         break;
4015                 }
4016         }
4017 }
4018
4019 int
4020 tcp_can_enable_pacing(void)
4021 {
4022
4023         if ((tcp_pacing_limit == -1) ||
4024             (tcp_pacing_limit > number_of_tcp_connections_pacing)) {
4025                 atomic_fetchadd_int(&number_of_tcp_connections_pacing, 1);
4026                 shadow_num_connections = number_of_tcp_connections_pacing;
4027                 return (1);
4028         } else {
4029                 counter_u64_add(tcp_pacing_failures, 1);
4030                 return (0);
4031         }
4032 }
4033
4034 static uint8_t tcp_pacing_warning = 0;
4035
4036 void
4037 tcp_decrement_paced_conn(void)
4038 {
4039         uint32_t ret;
4040
4041         ret = atomic_fetchadd_int(&number_of_tcp_connections_pacing, -1);
4042         shadow_num_connections = number_of_tcp_connections_pacing;
4043         KASSERT(ret != 0, ("tcp_paced_connection_exits -1 would cause wrap?"));
4044         if (ret == 0) {
4045                 if (tcp_pacing_limit != -1) {
4046                         printf("Warning all pacing is now disabled, count decrements invalidly!\n");
4047                         tcp_pacing_limit = 0;
4048                 } else if (tcp_pacing_warning == 0) {
4049                         printf("Warning pacing count is invalid, invalid decrement\n");
4050                         tcp_pacing_warning = 1;
4051                 }
4052         }
4053 }
4054
4055 static void
4056 tcp_default_switch_failed(struct tcpcb *tp)
4057 {
4058         /*
4059          * If a switch fails we only need to
4060          * care about two things:
4061          * a) The t_flags2
4062          * and
4063          * b) The timer granularity.
4064          * Timeouts, at least for now, don't use the
4065          * old callout system in the other stacks so
4066          * those are hopefully safe.
4067          */
4068         tcp_lro_features_off(tp);
4069         tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
4070 }
4071
4072 #ifdef TCP_ACCOUNTING
4073 int
4074 tcp_do_ack_accounting(struct tcpcb *tp, struct tcphdr *th, struct tcpopt *to, uint32_t tiwin, int mss)
4075 {
4076         if (SEQ_LT(th->th_ack, tp->snd_una)) {
4077                 /* Do we have a SACK? */
4078                 if (to->to_flags & TOF_SACK) {
4079                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4080                                 tp->tcp_cnt_counters[ACK_SACK]++;
4081                         }
4082                         return (ACK_SACK);
4083                 } else {
4084                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4085                                 tp->tcp_cnt_counters[ACK_BEHIND]++;
4086                         }
4087                         return (ACK_BEHIND);
4088                 }
4089         } else if (th->th_ack == tp->snd_una) {
4090                 /* Do we have a SACK? */
4091                 if (to->to_flags & TOF_SACK) {
4092                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4093                                 tp->tcp_cnt_counters[ACK_SACK]++;
4094                         }
4095                         return (ACK_SACK);
4096                 } else if (tiwin != tp->snd_wnd) {
4097                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4098                                 tp->tcp_cnt_counters[ACK_RWND]++;
4099                         }
4100                         return (ACK_RWND);
4101                 } else {
4102                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4103                                 tp->tcp_cnt_counters[ACK_DUPACK]++;
4104                         }
4105                         return (ACK_DUPACK);
4106                 }
4107         } else {
4108                 if (!SEQ_GT(th->th_ack, tp->snd_max)) {
4109                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4110                                 tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((th->th_ack - tp->snd_una) + mss - 1)/mss);
4111                         }
4112                 }
4113                 if (to->to_flags & TOF_SACK) {
4114                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4115                                 tp->tcp_cnt_counters[ACK_CUMACK_SACK]++;
4116                         }
4117                         return (ACK_CUMACK_SACK);
4118                 } else {
4119                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4120                                 tp->tcp_cnt_counters[ACK_CUMACK]++;
4121                         }
4122                         return (ACK_CUMACK);
4123                 }
4124         }
4125 }
4126 #endif
4127
4128 void
4129 tcp_change_time_units(struct tcpcb *tp, int granularity)
4130 {
4131         if (tp->t_tmr_granularity == granularity) {
4132                 /* We are there */
4133                 return;
4134         }
4135         if (granularity == TCP_TMR_GRANULARITY_USEC) {
4136                 KASSERT((tp->t_tmr_granularity == TCP_TMR_GRANULARITY_TICKS),
4137                         ("Granularity is not TICKS its %u in tp:%p",
4138                          tp->t_tmr_granularity, tp));
4139                 tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow);
4140                 if (tp->t_srtt > 1) {
4141                         uint32_t val, frac;
4142
4143                         val = tp->t_srtt >> TCP_RTT_SHIFT;
4144                         frac = tp->t_srtt & 0x1f;
4145                         tp->t_srtt = TICKS_2_USEC(val);
4146                         /*
4147                          * frac is the fractional part of the srtt (if any)
4148                          * but its in ticks and every bit represents
4149                          * 1/32nd of a hz.
4150                          */
4151                         if (frac) {
4152                                 if (hz == 1000) {
4153                                         frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
4154                                 } else {
4155                                         frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
4156                                 }
4157                                 tp->t_srtt += frac;
4158                         }
4159                 }
4160                 if (tp->t_rttvar) {
4161                         uint32_t val, frac;
4162
4163                         val = tp->t_rttvar >> TCP_RTTVAR_SHIFT;
4164                         frac = tp->t_rttvar & 0x1f;
4165                         tp->t_rttvar = TICKS_2_USEC(val);
4166                         /*
4167                          * frac is the fractional part of the srtt (if any)
4168                          * but its in ticks and every bit represents
4169                          * 1/32nd of a hz.
4170                          */
4171                         if (frac) {
4172                                 if (hz == 1000) {
4173                                         frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
4174                                 } else {
4175                                         frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
4176                                 }
4177                                 tp->t_rttvar += frac;
4178                         }
4179                 }
4180                 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_USEC;
4181         } else if (granularity == TCP_TMR_GRANULARITY_TICKS) {
4182                 /* Convert back to ticks, with  */
4183                 KASSERT((tp->t_tmr_granularity == TCP_TMR_GRANULARITY_USEC),
4184                         ("Granularity is not USEC its %u in tp:%p",
4185                          tp->t_tmr_granularity, tp));
4186                 if (tp->t_srtt > 1) {
4187                         uint32_t val, frac;
4188
4189                         val = USEC_2_TICKS(tp->t_srtt);
4190                         frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz);
4191                         tp->t_srtt = val << TCP_RTT_SHIFT;
4192                         /*
4193                          * frac is the fractional part here is left
4194                          * over from converting to hz and shifting.
4195                          * We need to convert this to the 5 bit
4196                          * remainder.
4197                          */
4198                         if (frac) {
4199                                 if (hz == 1000) {
4200                                         frac = (((uint64_t)frac *  (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
4201                                 } else {
4202                                         frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
4203                                 }
4204                                 tp->t_srtt += frac;
4205                         }
4206                 }
4207                 if (tp->t_rttvar) {
4208                         uint32_t val, frac;
4209
4210                         val = USEC_2_TICKS(tp->t_rttvar);
4211                         frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz);
4212                         tp->t_rttvar = val <<  TCP_RTTVAR_SHIFT;
4213                         /*
4214                          * frac is the fractional part here is left
4215                          * over from converting to hz and shifting.
4216                          * We need to convert this to the 5 bit
4217                          * remainder.
4218                          */
4219                         if (frac) {
4220                                 if (hz == 1000) {
4221                                         frac = (((uint64_t)frac *  (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
4222                                 } else {
4223                                         frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
4224                                 }
4225                                 tp->t_rttvar += frac;
4226                         }
4227                 }
4228                 tp->t_rttlow = USEC_2_TICKS(tp->t_rttlow);
4229                 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_TICKS;
4230         }
4231 #ifdef INVARIANTS
4232         else {
4233                 panic("Unknown granularity:%d tp:%p",
4234                       granularity, tp);
4235         }
4236 #endif
4237 }
4238
4239 void
4240 tcp_handle_orphaned_packets(struct tcpcb *tp)
4241 {
4242         struct mbuf *save, *m, *prev;
4243         /*
4244          * Called when a stack switch is occuring from the fini()
4245          * of the old stack. We assue the init() as already been
4246          * run of the new stack and it has set the t_flags2 to
4247          * what it supports. This function will then deal with any
4248          * differences i.e. cleanup packets that maybe queued that
4249          * the newstack does not support.
4250          */
4251
4252         if (tp->t_flags2 & TF2_MBUF_L_ACKS)
4253                 return;
4254         if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) == 0 &&
4255             !STAILQ_EMPTY(&tp->t_inqueue)) {
4256                 /*
4257                  * It is unsafe to process the packets since a
4258                  * reset may be lurking in them (its rare but it
4259                  * can occur). If we were to find a RST, then we
4260                  * would end up dropping the connection and the
4261                  * INP lock, so when we return the caller (tcp_usrreq)
4262                  * will blow up when it trys to unlock the inp.
4263                  * This new stack does not do any fancy LRO features
4264                  * so all we can do is toss the packets.
4265                  */
4266                 m = STAILQ_FIRST(&tp->t_inqueue);
4267                 STAILQ_INIT(&tp->t_inqueue);
4268                 STAILQ_FOREACH_FROM_SAFE(m, &tp->t_inqueue, m_stailqpkt, save)
4269                         m_freem(m);
4270         } else {
4271                 /*
4272                  * Here we have a stack that does mbuf queuing but
4273                  * does not support compressed ack's. We must
4274                  * walk all the mbufs and discard any compressed acks.
4275                  */
4276                 STAILQ_FOREACH_SAFE(m, &tp->t_inqueue, m_stailqpkt, save) {
4277                         if (m->m_flags & M_ACKCMP) {
4278                                 if (m == STAILQ_FIRST(&tp->t_inqueue))
4279                                         STAILQ_REMOVE_HEAD(&tp->t_inqueue,
4280                                             m_stailqpkt);
4281                                 else
4282                                         STAILQ_REMOVE_AFTER(&tp->t_inqueue,
4283                                             prev, m_stailqpkt);
4284                                 m_freem(m);
4285                         } else
4286                                 prev = m;
4287                 }
4288         }
4289 }
4290
4291 #ifdef TCP_REQUEST_TRK
4292 uint32_t
4293 tcp_estimate_tls_overhead(struct socket *so, uint64_t tls_usr_bytes)
4294 {
4295 #ifdef KERN_TLS
4296         struct ktls_session *tls;
4297         uint32_t rec_oh, records;
4298
4299         tls = so->so_snd.sb_tls_info;
4300         if (tls == NULL)
4301             return (0);
4302
4303         rec_oh = tls->params.tls_hlen + tls->params.tls_tlen;
4304         records = ((tls_usr_bytes + tls->params.max_frame_len - 1)/tls->params.max_frame_len);
4305         return (records * rec_oh);
4306 #else
4307         return (0);
4308 #endif
4309 }
4310
4311 extern uint32_t tcp_stale_entry_time;
4312 uint32_t tcp_stale_entry_time = 250000;
4313 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, usrlog_stale, CTLFLAG_RW,
4314     &tcp_stale_entry_time, 250000, "Time that a tcpreq entry without a sendfile ages out");
4315
4316 void
4317 tcp_req_log_req_info(struct tcpcb *tp, struct tcp_sendfile_track *req,
4318     uint16_t slot, uint8_t val, uint64_t offset, uint64_t nbytes)
4319 {
4320         if (tcp_bblogging_on(tp)) {
4321                 union tcp_log_stackspecific log;
4322                 struct timeval tv;
4323
4324                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
4325 #ifdef TCPHPTS
4326                 log.u_bbr.inhpts = tcp_in_hpts(tp);
4327 #endif
4328                 log.u_bbr.flex8 = val;
4329                 log.u_bbr.rttProp = req->timestamp;
4330                 log.u_bbr.delRate = req->start;
4331                 log.u_bbr.cur_del_rate = req->end;
4332                 log.u_bbr.flex1 = req->start_seq;
4333                 log.u_bbr.flex2 = req->end_seq;
4334                 log.u_bbr.flex3 = req->flags;
4335                 log.u_bbr.flex4 = ((req->localtime >> 32) & 0x00000000ffffffff);
4336                 log.u_bbr.flex5 = (req->localtime & 0x00000000ffffffff);
4337                 log.u_bbr.flex7 = slot;
4338                 log.u_bbr.bw_inuse = offset;
4339                 /* nbytes = flex6 | epoch */
4340                 log.u_bbr.flex6 = ((nbytes >> 32) & 0x00000000ffffffff);
4341                 log.u_bbr.epoch = (nbytes & 0x00000000ffffffff);
4342                 /* cspr =  lt_epoch | pkts_out */
4343                 log.u_bbr.lt_epoch = ((req->cspr >> 32) & 0x00000000ffffffff);
4344                 log.u_bbr.pkts_out |= (req->cspr & 0x00000000ffffffff);
4345                 log.u_bbr.applimited = tp->t_tcpreq_closed;
4346                 log.u_bbr.applimited <<= 8;
4347                 log.u_bbr.applimited |= tp->t_tcpreq_open;
4348                 log.u_bbr.applimited <<= 8;
4349                 log.u_bbr.applimited |= tp->t_tcpreq_req;
4350                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
4351                 TCP_LOG_EVENTP(tp, NULL,
4352                     &tptosocket(tp)->so_rcv,
4353                     &tptosocket(tp)->so_snd,
4354                     TCP_LOG_REQ_T, 0,
4355                     0, &log, false, &tv);
4356         }
4357 }
4358
4359 void
4360 tcp_req_free_a_slot(struct tcpcb *tp, struct tcp_sendfile_track *ent)
4361 {
4362         if (tp->t_tcpreq_req > 0)
4363                 tp->t_tcpreq_req--;
4364         if (ent->flags & TCP_TRK_TRACK_FLG_OPEN) {
4365                 if (tp->t_tcpreq_open > 0)
4366                         tp->t_tcpreq_open--;
4367         } else {
4368                 if (tp->t_tcpreq_closed > 0)
4369                         tp->t_tcpreq_closed--;
4370         }
4371         ent->flags = TCP_TRK_TRACK_FLG_EMPTY;
4372 }
4373
4374 static void
4375 tcp_req_check_for_stale_entries(struct tcpcb *tp, uint64_t ts, int rm_oldest)
4376 {
4377         struct tcp_sendfile_track *ent;
4378         uint64_t time_delta, oldest_delta;
4379         int i, oldest, oldest_set = 0, cnt_rm = 0;
4380
4381         for(i = 0; i < MAX_TCP_TRK_REQ; i++) {
4382                 ent = &tp->t_tcpreq_info[i];
4383                 if (ent->flags != TCP_TRK_TRACK_FLG_USED) {
4384                         /*
4385                          * We only care about closed end ranges
4386                          * that are allocated and have no sendfile
4387                          * ever touching them. They would be in
4388                          * state USED.
4389                          */
4390                         continue;
4391                 }
4392                 if (ts >= ent->localtime)
4393                         time_delta = ts - ent->localtime;
4394                 else
4395                         time_delta = 0;
4396                 if (time_delta &&
4397                     ((oldest_delta < time_delta) || (oldest_set == 0))) {
4398                         oldest_set = 1;
4399                         oldest = i;
4400                         oldest_delta = time_delta;
4401                 }
4402                 if (tcp_stale_entry_time && (time_delta >= tcp_stale_entry_time)) {
4403                         /*
4404                          * No sendfile in a our time-limit
4405                          * time to purge it.
4406                          */
4407                         cnt_rm++;
4408                         tcp_req_log_req_info(tp, &tp->t_tcpreq_info[i], i, TCP_TRK_REQ_LOG_STALE,
4409                                               time_delta, 0);
4410                         tcp_req_free_a_slot(tp, ent);
4411                 }
4412         }
4413         if ((cnt_rm == 0) && rm_oldest && oldest_set) {
4414                 ent = &tp->t_tcpreq_info[oldest];
4415                 tcp_req_log_req_info(tp, &tp->t_tcpreq_info[i], i, TCP_TRK_REQ_LOG_STALE,
4416                                       oldest_delta, 1);
4417                 tcp_req_free_a_slot(tp, ent);
4418         }
4419 }
4420
4421 int
4422 tcp_req_check_for_comp(struct tcpcb *tp, tcp_seq ack_point)
4423 {
4424         int i, ret=0;
4425         struct tcp_sendfile_track *ent;
4426
4427         /* Clean up any old closed end requests that are now completed */
4428         if (tp->t_tcpreq_req == 0)
4429                 return(0);
4430         if (tp->t_tcpreq_closed == 0)
4431                 return(0);
4432         for(i = 0; i < MAX_TCP_TRK_REQ; i++) {
4433                 ent = &tp->t_tcpreq_info[i];
4434                 /* Skip empty ones */
4435                 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY)
4436                         continue;
4437                 /* Skip open ones */
4438                 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN)
4439                         continue;
4440                 if (SEQ_GEQ(ack_point, ent->end_seq)) {
4441                         /* We are past it -- free it */
4442                         tcp_req_log_req_info(tp, ent,
4443                                               i, TCP_TRK_REQ_LOG_FREED, 0, 0);
4444                         tcp_req_free_a_slot(tp, ent);
4445                         ret++;
4446                 }
4447         }
4448         return (ret);
4449 }
4450
4451 int
4452 tcp_req_is_entry_comp(struct tcpcb *tp, struct tcp_sendfile_track *ent, tcp_seq ack_point)
4453 {
4454         if (tp->t_tcpreq_req == 0)
4455                 return(-1);
4456         if (tp->t_tcpreq_closed == 0)
4457                 return(-1);
4458         if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY)
4459                 return(-1);
4460         if (SEQ_GEQ(ack_point, ent->end_seq)) {
4461                 return (1);
4462         }
4463         return (0);
4464 }
4465
4466 struct tcp_sendfile_track *
4467 tcp_req_find_a_req_that_is_completed_by(struct tcpcb *tp, tcp_seq th_ack, int *ip)
4468 {
4469         /*
4470          * Given an ack point (th_ack) walk through our entries and
4471          * return the first one found that th_ack goes past the
4472          * end_seq.
4473          */
4474         struct tcp_sendfile_track *ent;
4475         int i;
4476
4477         if (tp->t_tcpreq_req == 0) {
4478                 /* none open */
4479                 return (NULL);
4480         }
4481         for(i = 0; i < MAX_TCP_TRK_REQ; i++) {
4482                 ent = &tp->t_tcpreq_info[i];
4483                 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY)
4484                         continue;
4485                 if ((ent->flags & TCP_TRK_TRACK_FLG_OPEN) == 0) {
4486                         if (SEQ_GEQ(th_ack, ent->end_seq)) {
4487                                 *ip = i;
4488                                 return (ent);
4489                         }
4490                 }
4491         }
4492         return (NULL);
4493 }
4494
4495 struct tcp_sendfile_track *
4496 tcp_req_find_req_for_seq(struct tcpcb *tp, tcp_seq seq)
4497 {
4498         struct tcp_sendfile_track *ent;
4499         int i;
4500
4501         if (tp->t_tcpreq_req == 0) {
4502                 /* none open */
4503                 return (NULL);
4504         }
4505         for(i = 0; i < MAX_TCP_TRK_REQ; i++) {
4506                 ent = &tp->t_tcpreq_info[i];
4507                 tcp_req_log_req_info(tp, ent, i, TCP_TRK_REQ_LOG_SEARCH,
4508                                       (uint64_t)seq, 0);
4509                 if (ent->flags == TCP_TRK_TRACK_FLG_EMPTY) {
4510                         continue;
4511                 }
4512                 if (ent->flags & TCP_TRK_TRACK_FLG_OPEN) {
4513                         /*
4514                          * An open end request only needs to
4515                          * match the beginning seq or be
4516                          * all we have (once we keep going on
4517                          * a open end request we may have a seq
4518                          * wrap).
4519                          */
4520                         if ((SEQ_GEQ(seq, ent->start_seq)) ||
4521                             (tp->t_tcpreq_closed == 0))
4522                                 return (ent);
4523                 } else {
4524                         /*
4525                          * For this one we need to
4526                          * be a bit more careful if its
4527                          * completed at least.
4528                          */
4529                         if ((SEQ_GEQ(seq, ent->start_seq)) &&
4530                             (SEQ_LT(seq, ent->end_seq))) {
4531                                 return (ent);
4532                         }
4533                 }
4534         }
4535         return (NULL);
4536 }
4537
4538 /* Should this be in its own file tcp_req.c ? */
4539 struct tcp_sendfile_track *
4540 tcp_req_alloc_req_full(struct tcpcb *tp, struct tcp_snd_req *req, uint64_t ts, int rec_dups)
4541 {
4542         struct tcp_sendfile_track *fil;
4543         int i, allocated;
4544
4545         /* In case the stack does not check for completions do so now */
4546         tcp_req_check_for_comp(tp, tp->snd_una);
4547         /* Check for stale entries */
4548         if (tp->t_tcpreq_req)
4549                 tcp_req_check_for_stale_entries(tp, ts,
4550                     (tp->t_tcpreq_req >= MAX_TCP_TRK_REQ));
4551         /* Check to see if this is a duplicate of one not started */
4552         if (tp->t_tcpreq_req) {
4553                 for(i = 0, allocated = 0; i < MAX_TCP_TRK_REQ; i++) {
4554                         fil = &tp->t_tcpreq_info[i];
4555                         if (fil->flags != TCP_TRK_TRACK_FLG_USED)
4556                                 continue;
4557                         if ((fil->timestamp == req->timestamp) &&
4558                             (fil->start == req->start) &&
4559                             ((fil->flags & TCP_TRK_TRACK_FLG_OPEN) ||
4560                              (fil->end == req->end))) {
4561                                 /*
4562                                  * We already have this request
4563                                  * and it has not been started with sendfile.
4564                                  * This probably means the user was returned
4565                                  * a 4xx of some sort and its going to age
4566                                  * out, lets not duplicate it.
4567                                  */
4568                                 return(fil);
4569                         }
4570                 }
4571         }
4572         /* Ok if there is no room at the inn we are in trouble */
4573         if (tp->t_tcpreq_req >= MAX_TCP_TRK_REQ) {
4574                 tcp_trace_point(tp, TCP_TP_REQ_LOG_FAIL);
4575                 for(i = 0; i < MAX_TCP_TRK_REQ; i++) {
4576                         tcp_req_log_req_info(tp, &tp->t_tcpreq_info[i],
4577                             i, TCP_TRK_REQ_LOG_ALLOCFAIL, 0, 0);
4578                 }
4579                 return (NULL);
4580         }
4581         for(i = 0, allocated = 0; i < MAX_TCP_TRK_REQ; i++) {
4582                 fil = &tp->t_tcpreq_info[i];
4583                 if (fil->flags == TCP_TRK_TRACK_FLG_EMPTY) {
4584                         allocated = 1;
4585                         fil->flags = TCP_TRK_TRACK_FLG_USED;
4586                         fil->timestamp = req->timestamp;
4587                         fil->localtime = ts;
4588                         fil->start = req->start;
4589                         if (req->flags & TCP_LOG_HTTPD_RANGE_END) {
4590                                 fil->end = req->end;
4591                         } else {
4592                                 fil->end = 0;
4593                                 fil->flags |= TCP_TRK_TRACK_FLG_OPEN;
4594                         }
4595                         /*
4596                          * We can set the min boundaries to the TCP Sequence space,
4597                          * but it might be found to be further up when sendfile
4598                          * actually runs on this range (if it ever does).
4599                          */
4600                         fil->sbcc_at_s = tptosocket(tp)->so_snd.sb_ccc;
4601                         fil->start_seq = tp->snd_una +
4602                             tptosocket(tp)->so_snd.sb_ccc;
4603                         fil->end_seq = (fil->start_seq + ((uint32_t)(fil->end - fil->start)));
4604                         if (tptosocket(tp)->so_snd.sb_tls_info) {
4605                                 /*
4606                                  * This session is doing TLS. Take a swag guess
4607                                  * at the overhead.
4608                                  */
4609                                 fil->end_seq += tcp_estimate_tls_overhead(
4610                                     tptosocket(tp), (fil->end - fil->start));
4611                         }
4612                         tp->t_tcpreq_req++;
4613                         if (fil->flags & TCP_TRK_TRACK_FLG_OPEN)
4614                                 tp->t_tcpreq_open++;
4615                         else
4616                                 tp->t_tcpreq_closed++;
4617                         tcp_req_log_req_info(tp, fil, i,
4618                             TCP_TRK_REQ_LOG_NEW, 0, 0);
4619                         break;
4620                 } else
4621                         fil = NULL;
4622         }
4623         return (fil);
4624 }
4625
4626 void
4627 tcp_req_alloc_req(struct tcpcb *tp, union tcp_log_userdata *user, uint64_t ts)
4628 {
4629         (void)tcp_req_alloc_req_full(tp, &user->tcp_req, ts, 1);
4630 }
4631 #endif
4632
4633 void
4634 tcp_log_socket_option(struct tcpcb *tp, uint32_t option_num, uint32_t option_val, int err)
4635 {
4636         if (tcp_bblogging_on(tp)) {
4637                 struct tcp_log_buffer *l;
4638
4639                 l = tcp_log_event(tp, NULL,
4640                         &tptosocket(tp)->so_rcv,
4641                         &tptosocket(tp)->so_snd,
4642                         TCP_LOG_SOCKET_OPT,
4643                         err, 0, NULL, 1,
4644                         NULL, NULL, 0, NULL);
4645                 if (l) {
4646                         l->tlb_flex1 = option_num;
4647                         l->tlb_flex2 = option_val;
4648                 }
4649         }
4650 }
4651
4652 uint32_t
4653 tcp_get_srtt(struct tcpcb *tp, int granularity)
4654 {
4655         uint32_t srtt;
4656
4657         KASSERT(granularity == TCP_TMR_GRANULARITY_USEC ||
4658             granularity == TCP_TMR_GRANULARITY_TICKS,
4659             ("%s: called with unexpected granularity %d", __func__,
4660             granularity));
4661
4662         srtt = tp->t_srtt;
4663
4664         /*
4665          * We only support two granularities. If the stored granularity
4666          * does not match the granularity requested by the caller,
4667          * convert the stored value to the requested unit of granularity.
4668          */
4669         if (tp->t_tmr_granularity != granularity) {
4670                 if (granularity == TCP_TMR_GRANULARITY_USEC)
4671                         srtt = TICKS_2_USEC(srtt);
4672                 else
4673                         srtt = USEC_2_TICKS(srtt);
4674         }
4675
4676         /*
4677          * If the srtt is stored with ticks granularity, we need to
4678          * unshift to get the actual value. We do this after the
4679          * conversion above (if one was necessary) in order to maximize
4680          * precision.
4681          */
4682         if (tp->t_tmr_granularity == TCP_TMR_GRANULARITY_TICKS)
4683                 srtt = srtt >> TCP_RTT_SHIFT;
4684
4685         return (srtt);
4686 }