sys/netinet/tcp_subr.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. Neither the name of the University nor the names of its contributors
  16  *    may be used to endorse or promote products derived from this software
  17  *    without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  *      @(#)tcp_subr.c  8.2 (Berkeley) 5/24/95
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_inet.h"
  38 #include "opt_inet6.h"
  39 #include "opt_ipsec.h"
  40 #include "opt_kern_tls.h"
  41
  42 #include <sys/param.h>
  43 #include <sys/systm.h>
  44 #include <sys/arb.h>
  45 #include <sys/callout.h>
  46 #include <sys/eventhandler.h>
  47 #ifdef TCP_HHOOK
  48 #include <sys/hhook.h>
  49 #endif
  50 #include <sys/kernel.h>
  51 #ifdef TCP_HHOOK
  52 #include <sys/khelp.h>
  53 #endif
  54 #ifdef KERN_TLS
  55 #include <sys/ktls.h>
  56 #endif
  57 #include <sys/qmath.h>
  58 #include <sys/stats.h>
  59 #include <sys/sysctl.h>
  60 #include <sys/jail.h>
  61 #include <sys/malloc.h>
  62 #include <sys/refcount.h>
  63 #include <sys/mbuf.h>
  64 #include <sys/priv.h>
  65 #include <sys/proc.h>
  66 #include <sys/sdt.h>
  67 #include <sys/socket.h>
  68 #include <sys/socketvar.h>
  69 #include <sys/protosw.h>
  70 #include <sys/random.h>
  71
  72 #include <vm/uma.h>
  73
  74 #include <net/route.h>
  75 #include <net/route/nhop.h>
  76 #include <net/if.h>
  77 #include <net/if_var.h>
  78 #include <net/if_private.h>
  79 #include <net/vnet.h>
  80
  81 #include <netinet/in.h>
  82 #include <netinet/in_fib.h>
  83 #include <netinet/in_kdtrace.h>
  84 #include <netinet/in_pcb.h>
  85 #include <netinet/in_systm.h>
  86 #include <netinet/in_var.h>
  87 #include <netinet/ip.h>
  88 #include <netinet/ip_icmp.h>
  89 #include <netinet/ip_var.h>
  90 #ifdef INET6
  91 #include <netinet/icmp6.h>
  92 #include <netinet/ip6.h>
  93 #include <netinet6/in6_fib.h>
  94 #include <netinet6/in6_pcb.h>
  95 #include <netinet6/ip6_var.h>
  96 #include <netinet6/scope6_var.h>
  97 #include <netinet6/nd6.h>
  98 #endif
  99
 100 #include <netinet/tcp.h>
 101 #ifdef INVARIANTS
 102 #define TCPSTATES
 103 #endif
 104 #include <netinet/tcp_fsm.h>
 105 #include <netinet/tcp_seq.h>
 106 #include <netinet/tcp_timer.h>
 107 #include <netinet/tcp_var.h>
 108 #include <netinet/tcp_ecn.h>
 109 #include <netinet/tcp_log_buf.h>
 110 #include <netinet/tcp_syncache.h>
 111 #include <netinet/tcp_hpts.h>
 112 #include <netinet/tcp_lro.h>
 113 #include <netinet/cc/cc.h>
 114 #include <netinet/tcpip.h>
 115 #include <netinet/tcp_fastopen.h>
 116 #include <netinet/tcp_accounting.h>
 117 #ifdef TCPPCAP
 118 #include <netinet/tcp_pcap.h>
 119 #endif
 120 #ifdef TCP_OFFLOAD
 121 #include <netinet/tcp_offload.h>
 122 #endif
 123 #include <netinet/udp.h>
 124 #include <netinet/udp_var.h>
 125 #ifdef INET6
 126 #include <netinet6/tcp6_var.h>
 127 #endif
 128
 129 #include <netipsec/ipsec_support.h>
 130
 131 #include <machine/in_cksum.h>
 132 #include <crypto/siphash/siphash.h>
 133
 134 #include <security/mac/mac_framework.h>
 135
 136 #ifdef INET6
 137 static ip6proto_ctlinput_t tcp6_ctlinput;
 138 static udp_tun_icmp_t tcp6_ctlinput_viaudp;
 139 #endif
 140
 141 VNET_DEFINE(int, tcp_mssdflt) = TCP_MSS;
 142 #ifdef INET6
 143 VNET_DEFINE(int, tcp_v6mssdflt) = TCP6_MSS;
 144 #endif
 145
 146 #ifdef TCP_SAD_DETECTION
 147 /*  Sack attack detection thresholds and such */
 148 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, sack_attack,
 149     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 150     "Sack Attack detection thresholds");
 151 int32_t tcp_force_detection = 0;
 152 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, force_detection,
 153     CTLFLAG_RW,
 154     &tcp_force_detection, 0,
 155     "Do we force detection even if the INP has it off?");
 156 int32_t tcp_sad_limit = 10000;
 157 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, limit,
 158     CTLFLAG_RW,
 159     &tcp_sad_limit, 10000,
 160     "If SaD is enabled, what is the limit to sendmap entries (0 = unlimited)?");
 161 int32_t tcp_sack_to_ack_thresh = 700;   /* 70 % */
 162 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sack_to_ack_thresh,
 163     CTLFLAG_RW,
 164     &tcp_sack_to_ack_thresh, 700,
 165     "Percentage of sacks to acks we must see above (10.1 percent is 101)?");
 166 int32_t tcp_sack_to_move_thresh = 600;  /* 60 % */
 167 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, move_thresh,
 168     CTLFLAG_RW,
 169     &tcp_sack_to_move_thresh, 600,
 170     "Percentage of sack moves we must see above (10.1 percent is 101)");
 171 int32_t tcp_restoral_thresh = 450;      /* 45 % (sack:2:ack -25%) (mv:ratio -15%) **/
 172 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, restore_thresh,
 173     CTLFLAG_RW,
 174     &tcp_restoral_thresh, 450,
 175     "Percentage of sack to ack percentage we must see below to restore(10.1 percent is 101)");
 176 int32_t tcp_sad_decay_val = 800;
 177 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, decay_per,
 178     CTLFLAG_RW,
 179     &tcp_sad_decay_val, 800,
 180     "The decay percentage (10.1 percent equals 101 )");
 181 int32_t tcp_map_minimum = 500;
 182 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, nummaps,
 183     CTLFLAG_RW,
 184     &tcp_map_minimum, 500,
 185     "Number of Map enteries before we start detection");
 186 int32_t tcp_sad_pacing_interval = 2000;
 187 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sad_pacing_int,
 188     CTLFLAG_RW,
 189     &tcp_sad_pacing_interval, 2000,
 190     "What is the minimum pacing interval for a classified attacker?");
 191
 192 int32_t tcp_sad_low_pps = 100;
 193 SYSCTL_INT(_net_inet_tcp_sack_attack, OID_AUTO, sad_low_pps,
 194     CTLFLAG_RW,
 195     &tcp_sad_low_pps, 100,
 196     "What is the input pps that below which we do not decay?");
 197 #endif
 198 uint32_t tcp_ack_war_time_window = 1000;
 199 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ack_war_timewindow,
 200     CTLFLAG_RW,
 201     &tcp_ack_war_time_window, 1000,
 202    "If the tcp_stack does ack-war prevention how many milliseconds are in its time window?");
 203 uint32_t tcp_ack_war_cnt = 5;
 204 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, ack_war_cnt,
 205     CTLFLAG_RW,
 206     &tcp_ack_war_cnt, 5,
 207    "If the tcp_stack does ack-war prevention how many acks can be sent in its time window?");
 208
 209 struct rwlock tcp_function_lock;
 210
 211 static int
 212 sysctl_net_inet_tcp_mss_check(SYSCTL_HANDLER_ARGS)
 213 {
 214         int error, new;
 215
 216         new = V_tcp_mssdflt;
 217         error = sysctl_handle_int(oidp, &new, 0, req);
 218         if (error == 0 && req->newptr) {
 219                 if (new < TCP_MINMSS)
 220                         error = EINVAL;
 221                 else
 222                         V_tcp_mssdflt = new;
 223         }
 224         return (error);
 225 }
 226
 227 SYSCTL_PROC(_net_inet_tcp, TCPCTL_MSSDFLT, mssdflt,
 228     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 229     &VNET_NAME(tcp_mssdflt), 0, &sysctl_net_inet_tcp_mss_check, "I",
 230     "Default TCP Maximum Segment Size");
 231
 232 #ifdef INET6
 233 static int
 234 sysctl_net_inet_tcp_mss_v6_check(SYSCTL_HANDLER_ARGS)
 235 {
 236         int error, new;
 237
 238         new = V_tcp_v6mssdflt;
 239         error = sysctl_handle_int(oidp, &new, 0, req);
 240         if (error == 0 && req->newptr) {
 241                 if (new < TCP_MINMSS)
 242                         error = EINVAL;
 243                 else
 244                         V_tcp_v6mssdflt = new;
 245         }
 246         return (error);
 247 }
 248
 249 SYSCTL_PROC(_net_inet_tcp, TCPCTL_V6MSSDFLT, v6mssdflt,
 250     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 251     &VNET_NAME(tcp_v6mssdflt), 0, &sysctl_net_inet_tcp_mss_v6_check, "I",
 252    "Default TCP Maximum Segment Size for IPv6");
 253 #endif /* INET6 */
 254
 255 /*
 256  * Minimum MSS we accept and use. This prevents DoS attacks where
 257  * we are forced to a ridiculous low MSS like 20 and send hundreds
 258  * of packets instead of one. The effect scales with the available
 259  * bandwidth and quickly saturates the CPU and network interface
 260  * with packet generation and sending. Set to zero to disable MINMSS
 261  * checking. This setting prevents us from sending too small packets.
 262  */
 263 VNET_DEFINE(int, tcp_minmss) = TCP_MINMSS;
 264 SYSCTL_INT(_net_inet_tcp, OID_AUTO, minmss, CTLFLAG_VNET | CTLFLAG_RW,
 265      &VNET_NAME(tcp_minmss), 0,
 266     "Minimum TCP Maximum Segment Size");
 267
 268 VNET_DEFINE(int, tcp_do_rfc1323) = 1;
 269 SYSCTL_INT(_net_inet_tcp, TCPCTL_DO_RFC1323, rfc1323, CTLFLAG_VNET | CTLFLAG_RW,
 270     &VNET_NAME(tcp_do_rfc1323), 0,
 271     "Enable rfc1323 (high performance TCP) extensions");
 272
 273 /*
 274  * As of June 2021, several TCP stacks violate RFC 7323 from September 2014.
 275  * Some stacks negotiate TS, but never send them after connection setup. Some
 276  * stacks negotiate TS, but don't send them when sending keep-alive segments.
 277  * These include modern widely deployed TCP stacks.
 278  * Therefore tolerating violations for now...
 279  */
 280 VNET_DEFINE(int, tcp_tolerate_missing_ts) = 1;
 281 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tolerate_missing_ts, CTLFLAG_VNET | CTLFLAG_RW,
 282     &VNET_NAME(tcp_tolerate_missing_ts), 0,
 283     "Tolerate missing TCP timestamps");
 284
 285 VNET_DEFINE(int, tcp_ts_offset_per_conn) = 1;
 286 SYSCTL_INT(_net_inet_tcp, OID_AUTO, ts_offset_per_conn, CTLFLAG_VNET | CTLFLAG_RW,
 287     &VNET_NAME(tcp_ts_offset_per_conn), 0,
 288     "Initialize TCP timestamps per connection instead of per host pair");
 289
 290 /* How many connections are pacing */
 291 static volatile uint32_t number_of_tcp_connections_pacing = 0;
 292 static uint32_t shadow_num_connections = 0;
 293 static counter_u64_t tcp_pacing_failures;
 294
 295 static int tcp_pacing_limit = 10000;
 296 SYSCTL_INT(_net_inet_tcp, OID_AUTO, pacing_limit, CTLFLAG_RW,
 297     &tcp_pacing_limit, 1000,
 298     "If the TCP stack does pacing, is there a limit (-1 = no, 0 = no pacing N = number of connections)");
 299
 300 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pacing_count, CTLFLAG_RD,
 301     &shadow_num_connections, 0, "Number of TCP connections being paced");
 302
 303 SYSCTL_COUNTER_U64(_net_inet_tcp, OID_AUTO, pacing_failures, CTLFLAG_RD,
 304     &tcp_pacing_failures, "Number of times we failed to enable pacing to avoid exceeding the limit");
 305
 306 static int      tcp_log_debug = 0;
 307 SYSCTL_INT(_net_inet_tcp, OID_AUTO, log_debug, CTLFLAG_RW,
 308     &tcp_log_debug, 0, "Log errors caused by incoming TCP segments");
 309
 310 static int      tcp_tcbhashsize;
 311 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcbhashsize, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
 312     &tcp_tcbhashsize, 0, "Size of TCP control-block hashtable");
 313
 314 static int      do_tcpdrain = 1;
 315 SYSCTL_INT(_net_inet_tcp, OID_AUTO, do_tcpdrain, CTLFLAG_RW, &do_tcpdrain, 0,
 316     "Enable tcp_drain routine for extra help when low on mbufs");
 317
 318 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, pcbcount, CTLFLAG_VNET | CTLFLAG_RD,
 319     &VNET_NAME(tcbinfo.ipi_count), 0, "Number of active PCBs");
 320
 321 VNET_DEFINE_STATIC(int, icmp_may_rst) = 1;
 322 #define V_icmp_may_rst                  VNET(icmp_may_rst)
 323 SYSCTL_INT(_net_inet_tcp, OID_AUTO, icmp_may_rst, CTLFLAG_VNET | CTLFLAG_RW,
 324     &VNET_NAME(icmp_may_rst), 0,
 325     "Certain ICMP unreachable messages may abort connections in SYN_SENT");
 326
 327 VNET_DEFINE_STATIC(int, tcp_isn_reseed_interval) = 0;
 328 #define V_tcp_isn_reseed_interval       VNET(tcp_isn_reseed_interval)
 329 SYSCTL_INT(_net_inet_tcp, OID_AUTO, isn_reseed_interval, CTLFLAG_VNET | CTLFLAG_RW,
 330     &VNET_NAME(tcp_isn_reseed_interval), 0,
 331     "Seconds between reseeding of ISN secret");
 332
 333 static int      tcp_soreceive_stream;
 334 SYSCTL_INT(_net_inet_tcp, OID_AUTO, soreceive_stream, CTLFLAG_RDTUN,
 335     &tcp_soreceive_stream, 0, "Using soreceive_stream for TCP sockets");
 336
 337 VNET_DEFINE(uma_zone_t, sack_hole_zone);
 338 #define V_sack_hole_zone                VNET(sack_hole_zone)
 339 VNET_DEFINE(uint32_t, tcp_map_entries_limit) = 0;       /* unlimited */
 340 static int
 341 sysctl_net_inet_tcp_map_limit_check(SYSCTL_HANDLER_ARGS)
 342 {
 343         int error;
 344         uint32_t new;
 345
 346         new = V_tcp_map_entries_limit;
 347         error = sysctl_handle_int(oidp, &new, 0, req);
 348         if (error == 0 && req->newptr) {
 349                 /* only allow "0" and value > minimum */
 350                 if (new > 0 && new < TCP_MIN_MAP_ENTRIES_LIMIT)
 351                         error = EINVAL;
 352                 else
 353                         V_tcp_map_entries_limit = new;
 354         }
 355         return (error);
 356 }
 357 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, map_limit,
 358     CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 359     &VNET_NAME(tcp_map_entries_limit), 0,
 360     &sysctl_net_inet_tcp_map_limit_check, "IU",
 361     "Total sendmap entries limit");
 362
 363 VNET_DEFINE(uint32_t, tcp_map_split_limit) = 0; /* unlimited */
 364 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, split_limit, CTLFLAG_VNET | CTLFLAG_RW,
 365      &VNET_NAME(tcp_map_split_limit), 0,
 366     "Total sendmap split entries limit");
 367
 368 #ifdef TCP_HHOOK
 369 VNET_DEFINE(struct hhook_head *, tcp_hhh[HHOOK_TCP_LAST+1]);
 370 #endif
 371
 372 #define TS_OFFSET_SECRET_LENGTH SIPHASH_KEY_LENGTH
 373 VNET_DEFINE_STATIC(u_char, ts_offset_secret[TS_OFFSET_SECRET_LENGTH]);
 374 #define V_ts_offset_secret      VNET(ts_offset_secret)
 375
 376 static int      tcp_default_fb_init(struct tcpcb *tp, void **ptr);
 377 static void     tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged);
 378 static int      tcp_default_handoff_ok(struct tcpcb *tp);
 379 static struct inpcb *tcp_notify(struct inpcb *, int);
 380 static struct inpcb *tcp_mtudisc_notify(struct inpcb *, int);
 381 static struct inpcb *tcp_mtudisc(struct inpcb *, int);
 382 static struct inpcb *tcp_drop_syn_sent(struct inpcb *, int);
 383 static char *   tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th,
 384                     const void *ip4hdr, const void *ip6hdr);
 385 static void     tcp_default_switch_failed(struct tcpcb *tp);
 386 static ipproto_ctlinput_t       tcp_ctlinput;
 387 static udp_tun_icmp_t           tcp_ctlinput_viaudp;
 388
 389 static struct tcp_function_block tcp_def_funcblk = {
 390         .tfb_tcp_block_name = "freebsd",
 391         .tfb_tcp_output = tcp_default_output,
 392         .tfb_tcp_do_segment = tcp_do_segment,
 393         .tfb_tcp_ctloutput = tcp_default_ctloutput,
 394         .tfb_tcp_handoff_ok = tcp_default_handoff_ok,
 395         .tfb_tcp_fb_init = tcp_default_fb_init,
 396         .tfb_tcp_fb_fini = tcp_default_fb_fini,
 397         .tfb_switch_failed = tcp_default_switch_failed,
 398 };
 399
 400 static int tcp_fb_cnt = 0;
 401 struct tcp_funchead t_functions;
 402 VNET_DEFINE_STATIC(struct tcp_function_block *, tcp_func_set_ptr) = &tcp_def_funcblk;
 403 #define V_tcp_func_set_ptr VNET(tcp_func_set_ptr)
 404
 405 void
 406 tcp_record_dsack(struct tcpcb *tp, tcp_seq start, tcp_seq end, int tlp)
 407 {
 408         TCPSTAT_INC(tcps_dsack_count);
 409         tp->t_dsack_pack++;
 410         if (tlp == 0) {
 411                 if (SEQ_GT(end, start)) {
 412                         tp->t_dsack_bytes += (end - start);
 413                         TCPSTAT_ADD(tcps_dsack_bytes, (end - start));
 414                 } else {
 415                         tp->t_dsack_tlp_bytes += (start - end);
 416                         TCPSTAT_ADD(tcps_dsack_bytes, (start - end));
 417                 }
 418         } else {
 419                 if (SEQ_GT(end, start)) {
 420                         tp->t_dsack_bytes += (end - start);
 421                         TCPSTAT_ADD(tcps_dsack_tlp_bytes, (end - start));
 422                 } else {
 423                         tp->t_dsack_tlp_bytes += (start - end);
 424                         TCPSTAT_ADD(tcps_dsack_tlp_bytes, (start - end));
 425                 }
 426         }
 427 }
 428
 429 static struct tcp_function_block *
 430 find_tcp_functions_locked(struct tcp_function_set *fs)
 431 {
 432         struct tcp_function *f;
 433         struct tcp_function_block *blk=NULL;
 434
 435         TAILQ_FOREACH(f, &t_functions, tf_next) {
 436                 if (strcmp(f->tf_name, fs->function_set_name) == 0) {
 437                         blk = f->tf_fb;
 438                         break;
 439                 }
 440         }
 441         return(blk);
 442 }
 443
 444 static struct tcp_function_block *
 445 find_tcp_fb_locked(struct tcp_function_block *blk, struct tcp_function **s)
 446 {
 447         struct tcp_function_block *rblk=NULL;
 448         struct tcp_function *f;
 449
 450         TAILQ_FOREACH(f, &t_functions, tf_next) {
 451                 if (f->tf_fb == blk) {
 452                         rblk = blk;
 453                         if (s) {
 454                                 *s = f;
 455                         }
 456                         break;
 457                 }
 458         }
 459         return (rblk);
 460 }
 461
 462 struct tcp_function_block *
 463 find_and_ref_tcp_functions(struct tcp_function_set *fs)
 464 {
 465         struct tcp_function_block *blk;
 466
 467         rw_rlock(&tcp_function_lock);
 468         blk = find_tcp_functions_locked(fs);
 469         if (blk)
 470                 refcount_acquire(&blk->tfb_refcnt);
 471         rw_runlock(&tcp_function_lock);
 472         return(blk);
 473 }
 474
 475 struct tcp_function_block *
 476 find_and_ref_tcp_fb(struct tcp_function_block *blk)
 477 {
 478         struct tcp_function_block *rblk;
 479
 480         rw_rlock(&tcp_function_lock);
 481         rblk = find_tcp_fb_locked(blk, NULL);
 482         if (rblk)
 483                 refcount_acquire(&rblk->tfb_refcnt);
 484         rw_runlock(&tcp_function_lock);
 485         return(rblk);
 486 }
 487
 488 /* Find a matching alias for the given tcp_function_block. */
 489 int
 490 find_tcp_function_alias(struct tcp_function_block *blk,
 491     struct tcp_function_set *fs)
 492 {
 493         struct tcp_function *f;
 494         int found;
 495
 496         found = 0;
 497         rw_rlock(&tcp_function_lock);
 498         TAILQ_FOREACH(f, &t_functions, tf_next) {
 499                 if ((f->tf_fb == blk) &&
 500                     (strncmp(f->tf_name, blk->tfb_tcp_block_name,
 501                         TCP_FUNCTION_NAME_LEN_MAX) != 0)) {
 502                         /* Matching function block with different name. */
 503                         strncpy(fs->function_set_name, f->tf_name,
 504                             TCP_FUNCTION_NAME_LEN_MAX);
 505                         found = 1;
 506                         break;
 507                 }
 508         }
 509         /* Null terminate the string appropriately. */
 510         if (found) {
 511                 fs->function_set_name[TCP_FUNCTION_NAME_LEN_MAX - 1] = '\0';
 512         } else {
 513                 fs->function_set_name[0] = '\0';
 514         }
 515         rw_runlock(&tcp_function_lock);
 516         return (found);
 517 }
 518
 519 static struct tcp_function_block *
 520 find_and_ref_tcp_default_fb(void)
 521 {
 522         struct tcp_function_block *rblk;
 523
 524         rw_rlock(&tcp_function_lock);
 525         rblk = V_tcp_func_set_ptr;
 526         refcount_acquire(&rblk->tfb_refcnt);
 527         rw_runlock(&tcp_function_lock);
 528         return (rblk);
 529 }
 530
 531 void
 532 tcp_switch_back_to_default(struct tcpcb *tp)
 533 {
 534         struct tcp_function_block *tfb;
 535         void *ptr = NULL;
 536
 537         KASSERT(tp->t_fb != &tcp_def_funcblk,
 538             ("%s: called by the built-in default stack", __func__));
 539
 540         /*
 541          * Now, we'll find a new function block to use.
 542          * Start by trying the current user-selected
 543          * default, unless this stack is the user-selected
 544          * default.
 545          */
 546         tfb = find_and_ref_tcp_default_fb();
 547         if (tfb == tp->t_fb) {
 548                 refcount_release(&tfb->tfb_refcnt);
 549                 tfb = NULL;
 550         }
 551         /* Does the stack accept this connection? */
 552         if (tfb != NULL && tfb->tfb_tcp_handoff_ok != NULL &&
 553             (*tfb->tfb_tcp_handoff_ok)(tp)) {
 554                 refcount_release(&tfb->tfb_refcnt);
 555                 tfb = NULL;
 556         }
 557         /* Try to use that stack. */
 558         if (tfb != NULL) {
 559                 /* Initialize the new stack. If it succeeds, we are done. */
 560                 if (tfb->tfb_tcp_fb_init == NULL ||
 561                     (*tfb->tfb_tcp_fb_init)(tp, &ptr) == 0) {
 562                         /* Release the old stack */
 563                         if (tp->t_fb->tfb_tcp_fb_fini != NULL)
 564                                 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
 565                         refcount_release(&tp->t_fb->tfb_refcnt);
 566                         /* Now set in all the pointers */
 567                         tp->t_fb = tfb;
 568                         tp->t_fb_ptr = ptr;
 569                         return;
 570                 }
 571                 /*
 572                  * Initialization failed. Release the reference count on
 573                  * the looked up default stack.
 574                  */
 575                 refcount_release(&tfb->tfb_refcnt);
 576         }
 577
 578         /*
 579          * If that wasn't feasible, use the built-in default
 580          * stack which is not allowed to reject anyone.
 581          */
 582         tfb = find_and_ref_tcp_fb(&tcp_def_funcblk);
 583         if (tfb == NULL) {
 584                 /* there always should be a default */
 585                 panic("Can't refer to tcp_def_funcblk");
 586         }
 587         if (tfb->tfb_tcp_handoff_ok != NULL) {
 588                 if ((*tfb->tfb_tcp_handoff_ok) (tp)) {
 589                         /* The default stack cannot say no */
 590                         panic("Default stack rejects a new session?");
 591                 }
 592         }
 593         if (tfb->tfb_tcp_fb_init != NULL &&
 594             (*tfb->tfb_tcp_fb_init)(tp, &ptr)) {
 595                 /* The default stack cannot fail */
 596                 panic("Default stack initialization failed");
 597         }
 598         /* Now release the old stack */
 599         if (tp->t_fb->tfb_tcp_fb_fini != NULL)
 600                 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 0);
 601         refcount_release(&tp->t_fb->tfb_refcnt);
 602         /* And set in the pointers to the new */
 603         tp->t_fb = tfb;
 604         tp->t_fb_ptr = ptr;
 605 }
 606
 607 static bool
 608 tcp_recv_udp_tunneled_packet(struct mbuf *m, int off, struct inpcb *inp,
 609     const struct sockaddr *sa, void *ctx)
 610 {
 611         struct ip *iph;
 612 #ifdef INET6
 613         struct ip6_hdr *ip6;
 614 #endif
 615         struct udphdr *uh;
 616         struct tcphdr *th;
 617         int thlen;
 618         uint16_t port;
 619
 620         TCPSTAT_INC(tcps_tunneled_pkts);
 621         if ((m->m_flags & M_PKTHDR) == 0) {
 622                 /* Can't handle one that is not a pkt hdr */
 623                 TCPSTAT_INC(tcps_tunneled_errs);
 624                 goto out;
 625         }
 626         thlen = sizeof(struct tcphdr);
 627         if (m->m_len < off + sizeof(struct udphdr) + thlen &&
 628             (m =  m_pullup(m, off + sizeof(struct udphdr) + thlen)) == NULL) {
 629                 TCPSTAT_INC(tcps_tunneled_errs);
 630                 goto out;
 631         }
 632         iph = mtod(m, struct ip *);
 633         uh = (struct udphdr *)((caddr_t)iph + off);
 634         th = (struct tcphdr *)(uh + 1);
 635         thlen = th->th_off << 2;
 636         if (m->m_len < off + sizeof(struct udphdr) + thlen) {
 637                 m =  m_pullup(m, off + sizeof(struct udphdr) + thlen);
 638                 if (m == NULL) {
 639                         TCPSTAT_INC(tcps_tunneled_errs);
 640                         goto out;
 641                 } else {
 642                         iph = mtod(m, struct ip *);
 643                         uh = (struct udphdr *)((caddr_t)iph + off);
 644                         th = (struct tcphdr *)(uh + 1);
 645                 }
 646         }
 647         m->m_pkthdr.tcp_tun_port = port = uh->uh_sport;
 648         bcopy(th, uh, m->m_len - off);
 649         m->m_len -= sizeof(struct udphdr);
 650         m->m_pkthdr.len -= sizeof(struct udphdr);
 651         /*
 652          * We use the same algorithm for
 653          * both UDP and TCP for c-sum. So
 654          * the code in tcp_input will skip
 655          * the checksum. So we do nothing
 656          * with the flag (m->m_pkthdr.csum_flags).
 657          */
 658         switch (iph->ip_v) {
 659 #ifdef INET
 660         case IPVERSION:
 661                 iph->ip_len = htons(ntohs(iph->ip_len) - sizeof(struct udphdr));
 662                 tcp_input_with_port(&m, &off, IPPROTO_TCP, port);
 663                 break;
 664 #endif
 665 #ifdef INET6
 666         case IPV6_VERSION >> 4:
 667                 ip6 = mtod(m, struct ip6_hdr *);
 668                 ip6->ip6_plen = htons(ntohs(ip6->ip6_plen) - sizeof(struct udphdr));
 669                 tcp6_input_with_port(&m, &off, IPPROTO_TCP, port);
 670                 break;
 671 #endif
 672         default:
 673                 goto out;
 674                 break;
 675         }
 676         return (true);
 677 out:
 678         m_freem(m);
 679
 680         return (true);
 681 }
 682
 683 static int
 684 sysctl_net_inet_default_tcp_functions(SYSCTL_HANDLER_ARGS)
 685 {
 686         int error=ENOENT;
 687         struct tcp_function_set fs;
 688         struct tcp_function_block *blk;
 689
 690         memset(&fs, 0, sizeof(fs));
 691         rw_rlock(&tcp_function_lock);
 692         blk = find_tcp_fb_locked(V_tcp_func_set_ptr, NULL);
 693         if (blk) {
 694                 /* Found him */
 695                 strcpy(fs.function_set_name, blk->tfb_tcp_block_name);
 696                 fs.pcbcnt = blk->tfb_refcnt;
 697         }
 698         rw_runlock(&tcp_function_lock);
 699         error = sysctl_handle_string(oidp, fs.function_set_name,
 700                                      sizeof(fs.function_set_name), req);
 701
 702         /* Check for error or no change */
 703         if (error != 0 || req->newptr == NULL)
 704                 return(error);
 705
 706         rw_wlock(&tcp_function_lock);
 707         blk = find_tcp_functions_locked(&fs);
 708         if ((blk == NULL) ||
 709             (blk->tfb_flags & TCP_FUNC_BEING_REMOVED)) {
 710                 error = ENOENT;
 711                 goto done;
 712         }
 713         V_tcp_func_set_ptr = blk;
 714 done:
 715         rw_wunlock(&tcp_function_lock);
 716         return (error);
 717 }
 718
 719 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_default,
 720     CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
 721     NULL, 0, sysctl_net_inet_default_tcp_functions, "A",
 722     "Set/get the default TCP functions");
 723
 724 static int
 725 sysctl_net_inet_list_available(SYSCTL_HANDLER_ARGS)
 726 {
 727         int error, cnt, linesz;
 728         struct tcp_function *f;
 729         char *buffer, *cp;
 730         size_t bufsz, outsz;
 731         bool alias;
 732
 733         cnt = 0;
 734         rw_rlock(&tcp_function_lock);
 735         TAILQ_FOREACH(f, &t_functions, tf_next) {
 736                 cnt++;
 737         }
 738         rw_runlock(&tcp_function_lock);
 739
 740         bufsz = (cnt+2) * ((TCP_FUNCTION_NAME_LEN_MAX * 2) + 13) + 1;
 741         buffer = malloc(bufsz, M_TEMP, M_WAITOK);
 742
 743         error = 0;
 744         cp = buffer;
 745
 746         linesz = snprintf(cp, bufsz, "\n%-32s%c %-32s %s\n", "Stack", 'D',
 747             "Alias", "PCB count");
 748         cp += linesz;
 749         bufsz -= linesz;
 750         outsz = linesz;
 751
 752         rw_rlock(&tcp_function_lock);
 753         TAILQ_FOREACH(f, &t_functions, tf_next) {
 754                 alias = (f->tf_name != f->tf_fb->tfb_tcp_block_name);
 755                 linesz = snprintf(cp, bufsz, "%-32s%c %-32s %u\n",
 756                     f->tf_fb->tfb_tcp_block_name,
 757                     (f->tf_fb == V_tcp_func_set_ptr) ? '*' : ' ',
 758                     alias ? f->tf_name : "-",
 759                     f->tf_fb->tfb_refcnt);
 760                 if (linesz >= bufsz) {
 761                         error = EOVERFLOW;
 762                         break;
 763                 }
 764                 cp += linesz;
 765                 bufsz -= linesz;
 766                 outsz += linesz;
 767         }
 768         rw_runlock(&tcp_function_lock);
 769         if (error == 0)
 770                 error = sysctl_handle_string(oidp, buffer, outsz + 1, req);
 771         free(buffer, M_TEMP);
 772         return (error);
 773 }
 774
 775 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, functions_available,
 776     CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
 777     NULL, 0, sysctl_net_inet_list_available, "A",
 778     "list available TCP Function sets");
 779
 780 VNET_DEFINE(int, tcp_udp_tunneling_port) = TCP_TUNNELING_PORT_DEFAULT;
 781
 782 #ifdef INET
 783 VNET_DEFINE(struct socket *, udp4_tun_socket) = NULL;
 784 #define V_udp4_tun_socket       VNET(udp4_tun_socket)
 785 #endif
 786 #ifdef INET6
 787 VNET_DEFINE(struct socket *, udp6_tun_socket) = NULL;
 788 #define V_udp6_tun_socket       VNET(udp6_tun_socket)
 789 #endif
 790
 791 static struct sx tcpoudp_lock;
 792
 793 static void
 794 tcp_over_udp_stop(void)
 795 {
 796
 797         sx_assert(&tcpoudp_lock, SA_XLOCKED);
 798
 799 #ifdef INET
 800         if (V_udp4_tun_socket != NULL) {
 801                 soclose(V_udp4_tun_socket);
 802                 V_udp4_tun_socket = NULL;
 803         }
 804 #endif
 805 #ifdef INET6
 806         if (V_udp6_tun_socket != NULL) {
 807                 soclose(V_udp6_tun_socket);
 808                 V_udp6_tun_socket = NULL;
 809         }
 810 #endif
 811 }
 812
 813 static int
 814 tcp_over_udp_start(void)
 815 {
 816         uint16_t port;
 817         int ret;
 818 #ifdef INET
 819         struct sockaddr_in sin;
 820 #endif
 821 #ifdef INET6
 822         struct sockaddr_in6 sin6;
 823 #endif
 824
 825         sx_assert(&tcpoudp_lock, SA_XLOCKED);
 826
 827         port = V_tcp_udp_tunneling_port;
 828         if (ntohs(port) == 0) {
 829                 /* Must have a port set */
 830                 return (EINVAL);
 831         }
 832 #ifdef INET
 833         if (V_udp4_tun_socket != NULL) {
 834                 /* Already running -- must stop first */
 835                 return (EALREADY);
 836         }
 837 #endif
 838 #ifdef INET6
 839         if (V_udp6_tun_socket != NULL) {
 840                 /* Already running -- must stop first */
 841                 return (EALREADY);
 842         }
 843 #endif
 844 #ifdef INET
 845         if ((ret = socreate(PF_INET, &V_udp4_tun_socket,
 846             SOCK_DGRAM, IPPROTO_UDP,
 847             curthread->td_ucred, curthread))) {
 848                 tcp_over_udp_stop();
 849                 return (ret);
 850         }
 851         /* Call the special UDP hook. */
 852         if ((ret = udp_set_kernel_tunneling(V_udp4_tun_socket,
 853             tcp_recv_udp_tunneled_packet,
 854             tcp_ctlinput_viaudp,
 855             NULL))) {
 856                 tcp_over_udp_stop();
 857                 return (ret);
 858         }
 859         /* Ok, we have a socket, bind it to the port. */
 860         memset(&sin, 0, sizeof(struct sockaddr_in));
 861         sin.sin_len = sizeof(struct sockaddr_in);
 862         sin.sin_family = AF_INET;
 863         sin.sin_port = htons(port);
 864         if ((ret = sobind(V_udp4_tun_socket,
 865             (struct sockaddr *)&sin, curthread))) {
 866                 tcp_over_udp_stop();
 867                 return (ret);
 868         }
 869 #endif
 870 #ifdef INET6
 871         if ((ret = socreate(PF_INET6, &V_udp6_tun_socket,
 872             SOCK_DGRAM, IPPROTO_UDP,
 873             curthread->td_ucred, curthread))) {
 874                 tcp_over_udp_stop();
 875                 return (ret);
 876         }
 877         /* Call the special UDP hook. */
 878         if ((ret = udp_set_kernel_tunneling(V_udp6_tun_socket,
 879             tcp_recv_udp_tunneled_packet,
 880             tcp6_ctlinput_viaudp,
 881             NULL))) {
 882                 tcp_over_udp_stop();
 883                 return (ret);
 884         }
 885         /* Ok, we have a socket, bind it to the port. */
 886         memset(&sin6, 0, sizeof(struct sockaddr_in6));
 887         sin6.sin6_len = sizeof(struct sockaddr_in6);
 888         sin6.sin6_family = AF_INET6;
 889         sin6.sin6_port = htons(port);
 890         if ((ret = sobind(V_udp6_tun_socket,
 891             (struct sockaddr *)&sin6, curthread))) {
 892                 tcp_over_udp_stop();
 893                 return (ret);
 894         }
 895 #endif
 896         return (0);
 897 }
 898
 899 static int
 900 sysctl_net_inet_tcp_udp_tunneling_port_check(SYSCTL_HANDLER_ARGS)
 901 {
 902         int error;
 903         uint32_t old, new;
 904
 905         old = V_tcp_udp_tunneling_port;
 906         new = old;
 907         error = sysctl_handle_int(oidp, &new, 0, req);
 908         if ((error == 0) &&
 909             (req->newptr != NULL)) {
 910                 if ((new < TCP_TUNNELING_PORT_MIN) ||
 911                     (new > TCP_TUNNELING_PORT_MAX)) {
 912                         error = EINVAL;
 913                 } else {
 914                         sx_xlock(&tcpoudp_lock);
 915                         V_tcp_udp_tunneling_port = new;
 916                         if (old != 0) {
 917                                 tcp_over_udp_stop();
 918                         }
 919                         if (new != 0) {
 920                                 error = tcp_over_udp_start();
 921                                 if (error != 0) {
 922                                         V_tcp_udp_tunneling_port = 0;
 923                                 }
 924                         }
 925                         sx_xunlock(&tcpoudp_lock);
 926                 }
 927         }
 928         return (error);
 929 }
 930
 931 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, udp_tunneling_port,
 932     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 933     &VNET_NAME(tcp_udp_tunneling_port),
 934     0, &sysctl_net_inet_tcp_udp_tunneling_port_check, "IU",
 935     "Tunneling port for tcp over udp");
 936
 937 VNET_DEFINE(int, tcp_udp_tunneling_overhead) = TCP_TUNNELING_OVERHEAD_DEFAULT;
 938
 939 static int
 940 sysctl_net_inet_tcp_udp_tunneling_overhead_check(SYSCTL_HANDLER_ARGS)
 941 {
 942         int error, new;
 943
 944         new = V_tcp_udp_tunneling_overhead;
 945         error = sysctl_handle_int(oidp, &new, 0, req);
 946         if (error == 0 && req->newptr) {
 947                 if ((new < TCP_TUNNELING_OVERHEAD_MIN) ||
 948                     (new > TCP_TUNNELING_OVERHEAD_MAX))
 949                         error = EINVAL;
 950                 else
 951                         V_tcp_udp_tunneling_overhead = new;
 952         }
 953         return (error);
 954 }
 955
 956 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, udp_tunneling_overhead,
 957     CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
 958     &VNET_NAME(tcp_udp_tunneling_overhead),
 959     0, &sysctl_net_inet_tcp_udp_tunneling_overhead_check, "IU",
 960     "MSS reduction when using tcp over udp");
 961
 962 /*
 963  * Exports one (struct tcp_function_info) for each alias/name.
 964  */
 965 static int
 966 sysctl_net_inet_list_func_info(SYSCTL_HANDLER_ARGS)
 967 {
 968         int cnt, error;
 969         struct tcp_function *f;
 970         struct tcp_function_info tfi;
 971
 972         /*
 973          * We don't allow writes.
 974          */
 975         if (req->newptr != NULL)
 976                 return (EINVAL);
 977
 978         /*
 979          * Wire the old buffer so we can directly copy the functions to
 980          * user space without dropping the lock.
 981          */
 982         if (req->oldptr != NULL) {
 983                 error = sysctl_wire_old_buffer(req, 0);
 984                 if (error)
 985                         return (error);
 986         }
 987
 988         /*
 989          * Walk the list and copy out matching entries. If INVARIANTS
 990          * is compiled in, also walk the list to verify the length of
 991          * the list matches what we have recorded.
 992          */
 993         rw_rlock(&tcp_function_lock);
 994
 995         cnt = 0;
 996 #ifndef INVARIANTS
 997         if (req->oldptr == NULL) {
 998                 cnt = tcp_fb_cnt;
 999                 goto skip_loop;
1000         }
1001 #endif
1002         TAILQ_FOREACH(f, &t_functions, tf_next) {
1003 #ifdef INVARIANTS
1004                 cnt++;
1005 #endif
1006                 if (req->oldptr != NULL) {
1007                         bzero(&tfi, sizeof(tfi));
1008                         tfi.tfi_refcnt = f->tf_fb->tfb_refcnt;
1009                         tfi.tfi_id = f->tf_fb->tfb_id;
1010                         (void)strlcpy(tfi.tfi_alias, f->tf_name,
1011                             sizeof(tfi.tfi_alias));
1012                         (void)strlcpy(tfi.tfi_name,
1013                             f->tf_fb->tfb_tcp_block_name, sizeof(tfi.tfi_name));
1014                         error = SYSCTL_OUT(req, &tfi, sizeof(tfi));
1015                         /*
1016                          * Don't stop on error, as that is the
1017                          * mechanism we use to accumulate length
1018                          * information if the buffer was too short.
1019                          */
1020                 }
1021         }
1022         KASSERT(cnt == tcp_fb_cnt,
1023             ("%s: cnt (%d) != tcp_fb_cnt (%d)", __func__, cnt, tcp_fb_cnt));
1024 #ifndef INVARIANTS
1025 skip_loop:
1026 #endif
1027         rw_runlock(&tcp_function_lock);
1028         if (req->oldptr == NULL)
1029                 error = SYSCTL_OUT(req, NULL,
1030                     (cnt + 1) * sizeof(struct tcp_function_info));
1031
1032         return (error);
1033 }
1034
1035 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, function_info,
1036             CTLTYPE_OPAQUE | CTLFLAG_SKIP | CTLFLAG_RD | CTLFLAG_MPSAFE,
1037             NULL, 0, sysctl_net_inet_list_func_info, "S,tcp_function_info",
1038             "List TCP function block name-to-ID mappings");
1039
1040 /*
1041  * tfb_tcp_handoff_ok() function for the default stack.
1042  * Note that we'll basically try to take all comers.
1043  */
1044 static int
1045 tcp_default_handoff_ok(struct tcpcb *tp)
1046 {
1047
1048         return (0);
1049 }
1050
1051 /*
1052  * tfb_tcp_fb_init() function for the default stack.
1053  *
1054  * This handles making sure we have appropriate timers set if you are
1055  * transitioning a socket that has some amount of setup done.
1056  *
1057  * The init() fuction from the default can *never* return non-zero i.e.
1058  * it is required to always succeed since it is the stack of last resort!
1059  */
1060 static int
1061 tcp_default_fb_init(struct tcpcb *tp, void **ptr)
1062 {
1063         struct socket *so = tptosocket(tp);
1064         int rexmt;
1065
1066         INP_WLOCK_ASSERT(tptoinpcb(tp));
1067         /* We don't use the pointer */
1068         *ptr = NULL;
1069
1070         KASSERT(tp->t_state >= 0 && tp->t_state < TCPS_TIME_WAIT,
1071             ("%s: connection %p in unexpected state %d", __func__, tp,
1072             tp->t_state));
1073
1074         /* Make sure we get no interesting mbuf queuing behavior */
1075         /* All mbuf queue/ack compress flags should be off */
1076         tcp_lro_features_off(tp);
1077
1078         /* Cancel the GP measurement in progress */
1079         tp->t_flags &= ~TF_GPUTINPROG;
1080         /* Validate the timers are not in usec, if they are convert */
1081         tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
1082         if ((tp->t_state == TCPS_SYN_SENT) ||
1083             (tp->t_state == TCPS_SYN_RECEIVED))
1084                 rexmt = tcp_rexmit_initial * tcp_backoff[tp->t_rxtshift];
1085         else
1086                 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1087         if (tp->t_rxtshift == 0)
1088                 tp->t_rxtcur = rexmt;
1089         else
1090                 TCPT_RANGESET(tp->t_rxtcur, rexmt, tp->t_rttmin, TCPTV_REXMTMAX);
1091
1092         /*
1093          * Nothing to do for ESTABLISHED or LISTEN states. And, we don't
1094          * know what to do for unexpected states (which includes TIME_WAIT).
1095          */
1096         if (tp->t_state <= TCPS_LISTEN || tp->t_state >= TCPS_TIME_WAIT)
1097                 return (0);
1098
1099         /*
1100          * Make sure some kind of transmission timer is set if there is
1101          * outstanding data.
1102          */
1103         if ((!TCPS_HAVEESTABLISHED(tp->t_state) || sbavail(&so->so_snd) ||
1104             tp->snd_una != tp->snd_max) && !(tcp_timer_active(tp, TT_REXMT) ||
1105             tcp_timer_active(tp, TT_PERSIST))) {
1106                 /*
1107                  * If the session has established and it looks like it should
1108                  * be in the persist state, set the persist timer. Otherwise,
1109                  * set the retransmit timer.
1110                  */
1111                 if (TCPS_HAVEESTABLISHED(tp->t_state) && tp->snd_wnd == 0 &&
1112                     (int32_t)(tp->snd_nxt - tp->snd_una) <
1113                     (int32_t)sbavail(&so->so_snd))
1114                         tcp_setpersist(tp);
1115                 else
1116                         tcp_timer_activate(tp, TT_REXMT, TP_RXTCUR(tp));
1117         }
1118
1119         /* All non-embryonic sessions get a keepalive timer. */
1120         if (!tcp_timer_active(tp, TT_KEEP))
1121                 tcp_timer_activate(tp, TT_KEEP,
1122                     TCPS_HAVEESTABLISHED(tp->t_state) ? TP_KEEPIDLE(tp) :
1123                     TP_KEEPINIT(tp));
1124
1125         /*
1126          * Make sure critical variables are initialized
1127          * if transitioning while in Recovery.
1128          */
1129         if IN_FASTRECOVERY(tp->t_flags) {
1130                 if (tp->sackhint.recover_fs == 0)
1131                         tp->sackhint.recover_fs = max(1,
1132                             tp->snd_nxt - tp->snd_una);
1133         }
1134
1135         return (0);
1136 }
1137
1138 /*
1139  * tfb_tcp_fb_fini() function for the default stack.
1140  *
1141  * This changes state as necessary (or prudent) to prepare for another stack
1142  * to assume responsibility for the connection.
1143  */
1144 static void
1145 tcp_default_fb_fini(struct tcpcb *tp, int tcb_is_purged)
1146 {
1147
1148         INP_WLOCK_ASSERT(tptoinpcb(tp));
1149
1150 #ifdef TCP_BLACKBOX
1151         tcp_log_flowend(tp);
1152 #endif
1153         tp->t_acktime = 0;
1154         return;
1155 }
1156
1157 /*
1158  * Target size of TCP PCB hash tables. Must be a power of two.
1159  *
1160  * Note that this can be overridden by the kernel environment
1161  * variable net.inet.tcp.tcbhashsize
1162  */
1163 #ifndef TCBHASHSIZE
1164 #define TCBHASHSIZE     0
1165 #endif
1166
1167 MALLOC_DEFINE(M_TCPLOG, "tcplog", "TCP address and flags print buffers");
1168 MALLOC_DEFINE(M_TCPFUNCTIONS, "tcpfunc", "TCP function set memory");
1169
1170 static struct mtx isn_mtx;
1171
1172 #define ISN_LOCK_INIT() mtx_init(&isn_mtx, "isn_mtx", NULL, MTX_DEF)
1173 #define ISN_LOCK()      mtx_lock(&isn_mtx)
1174 #define ISN_UNLOCK()    mtx_unlock(&isn_mtx)
1175
1176 INPCBSTORAGE_DEFINE(tcpcbstor, tcpcb, "tcpinp", "tcp_inpcb", "tcp", "tcphash");
1177
1178 /*
1179  * Take a value and get the next power of 2 that doesn't overflow.
1180  * Used to size the tcp_inpcb hash buckets.
1181  */
1182 static int
1183 maketcp_hashsize(int size)
1184 {
1185         int hashsize;
1186
1187         /*
1188          * auto tune.
1189          * get the next power of 2 higher than maxsockets.
1190          */
1191         hashsize = 1 << fls(size);
1192         /* catch overflow, and just go one power of 2 smaller */
1193         if (hashsize < size) {
1194                 hashsize = 1 << (fls(size) - 1);
1195         }
1196         return (hashsize);
1197 }
1198
1199 static volatile int next_tcp_stack_id = 1;
1200
1201 /*
1202  * Register a TCP function block with the name provided in the names
1203  * array.  (Note that this function does NOT automatically register
1204  * blk->tfb_tcp_block_name as a stack name.  Therefore, you should
1205  * explicitly include blk->tfb_tcp_block_name in the list of names if
1206  * you wish to register the stack with that name.)
1207  *
1208  * Either all name registrations will succeed or all will fail.  If
1209  * a name registration fails, the function will update the num_names
1210  * argument to point to the array index of the name that encountered
1211  * the failure.
1212  *
1213  * Returns 0 on success, or an error code on failure.
1214  */
1215 int
1216 register_tcp_functions_as_names(struct tcp_function_block *blk, int wait,
1217     const char *names[], int *num_names)
1218 {
1219         struct tcp_function *n;
1220         struct tcp_function_set fs;
1221         int error, i;
1222
1223         KASSERT(names != NULL && *num_names > 0,
1224             ("%s: Called with 0-length name list", __func__));
1225         KASSERT(names != NULL, ("%s: Called with NULL name list", __func__));
1226         KASSERT(rw_initialized(&tcp_function_lock),
1227             ("%s: called too early", __func__));
1228
1229         if ((blk->tfb_tcp_output == NULL) ||
1230             (blk->tfb_tcp_do_segment == NULL) ||
1231             (blk->tfb_tcp_ctloutput == NULL) ||
1232             (strlen(blk->tfb_tcp_block_name) == 0)) {
1233                 /*
1234                  * These functions are required and you
1235                  * need a name.
1236                  */
1237                 *num_names = 0;
1238                 return (EINVAL);
1239         }
1240
1241         if (blk->tfb_flags & TCP_FUNC_BEING_REMOVED) {
1242                 *num_names = 0;
1243                 return (EINVAL);
1244         }
1245
1246         refcount_init(&blk->tfb_refcnt, 0);
1247         blk->tfb_id = atomic_fetchadd_int(&next_tcp_stack_id, 1);
1248         for (i = 0; i < *num_names; i++) {
1249                 n = malloc(sizeof(struct tcp_function), M_TCPFUNCTIONS, wait);
1250                 if (n == NULL) {
1251                         error = ENOMEM;
1252                         goto cleanup;
1253                 }
1254                 n->tf_fb = blk;
1255
1256                 (void)strlcpy(fs.function_set_name, names[i],
1257                     sizeof(fs.function_set_name));
1258                 rw_wlock(&tcp_function_lock);
1259                 if (find_tcp_functions_locked(&fs) != NULL) {
1260                         /* Duplicate name space not allowed */
1261                         rw_wunlock(&tcp_function_lock);
1262                         free(n, M_TCPFUNCTIONS);
1263                         error = EALREADY;
1264                         goto cleanup;
1265                 }
1266                 (void)strlcpy(n->tf_name, names[i], sizeof(n->tf_name));
1267                 TAILQ_INSERT_TAIL(&t_functions, n, tf_next);
1268                 tcp_fb_cnt++;
1269                 rw_wunlock(&tcp_function_lock);
1270         }
1271         return(0);
1272
1273 cleanup:
1274         /*
1275          * Deregister the names we just added. Because registration failed
1276          * for names[i], we don't need to deregister that name.
1277          */
1278         *num_names = i;
1279         rw_wlock(&tcp_function_lock);
1280         while (--i >= 0) {
1281                 TAILQ_FOREACH(n, &t_functions, tf_next) {
1282                         if (!strncmp(n->tf_name, names[i],
1283                             TCP_FUNCTION_NAME_LEN_MAX)) {
1284                                 TAILQ_REMOVE(&t_functions, n, tf_next);
1285                                 tcp_fb_cnt--;
1286                                 n->tf_fb = NULL;
1287                                 free(n, M_TCPFUNCTIONS);
1288                                 break;
1289                         }
1290                 }
1291         }
1292         rw_wunlock(&tcp_function_lock);
1293         return (error);
1294 }
1295
1296 /*
1297  * Register a TCP function block using the name provided in the name
1298  * argument.
1299  *
1300  * Returns 0 on success, or an error code on failure.
1301  */
1302 int
1303 register_tcp_functions_as_name(struct tcp_function_block *blk, const char *name,
1304     int wait)
1305 {
1306         const char *name_list[1];
1307         int num_names, rv;
1308
1309         num_names = 1;
1310         if (name != NULL)
1311                 name_list[0] = name;
1312         else
1313                 name_list[0] = blk->tfb_tcp_block_name;
1314         rv = register_tcp_functions_as_names(blk, wait, name_list, &num_names);
1315         return (rv);
1316 }
1317
1318 /*
1319  * Register a TCP function block using the name defined in
1320  * blk->tfb_tcp_block_name.
1321  *
1322  * Returns 0 on success, or an error code on failure.
1323  */
1324 int
1325 register_tcp_functions(struct tcp_function_block *blk, int wait)
1326 {
1327
1328         return (register_tcp_functions_as_name(blk, NULL, wait));
1329 }
1330
1331 /*
1332  * Deregister all names associated with a function block. This
1333  * functionally removes the function block from use within the system.
1334  *
1335  * When called with a true quiesce argument, mark the function block
1336  * as being removed so no more stacks will use it and determine
1337  * whether the removal would succeed.
1338  *
1339  * When called with a false quiesce argument, actually attempt the
1340  * removal.
1341  *
1342  * When called with a force argument, attempt to switch all TCBs to
1343  * use the default stack instead of returning EBUSY.
1344  *
1345  * Returns 0 on success (or if the removal would succeed), or an error
1346  * code on failure.
1347  */
1348 int
1349 deregister_tcp_functions(struct tcp_function_block *blk, bool quiesce,
1350     bool force)
1351 {
1352         struct tcp_function *f;
1353         VNET_ITERATOR_DECL(vnet_iter);
1354
1355         if (blk == &tcp_def_funcblk) {
1356                 /* You can't un-register the default */
1357                 return (EPERM);
1358         }
1359         rw_wlock(&tcp_function_lock);
1360         VNET_LIST_RLOCK_NOSLEEP();
1361         VNET_FOREACH(vnet_iter) {
1362                 CURVNET_SET(vnet_iter);
1363                 if (blk == V_tcp_func_set_ptr) {
1364                         /* You can't free the current default in some vnet. */
1365                         CURVNET_RESTORE();
1366                         VNET_LIST_RUNLOCK_NOSLEEP();
1367                         rw_wunlock(&tcp_function_lock);
1368                         return (EBUSY);
1369                 }
1370                 CURVNET_RESTORE();
1371         }
1372         VNET_LIST_RUNLOCK_NOSLEEP();
1373         /* Mark the block so no more stacks can use it. */
1374         blk->tfb_flags |= TCP_FUNC_BEING_REMOVED;
1375         /*
1376          * If TCBs are still attached to the stack, attempt to switch them
1377          * to the default stack.
1378          */
1379         if (force && blk->tfb_refcnt) {
1380                 struct inpcb *inp;
1381                 struct tcpcb *tp;
1382                 VNET_ITERATOR_DECL(vnet_iter);
1383
1384                 rw_wunlock(&tcp_function_lock);
1385
1386                 VNET_LIST_RLOCK();
1387                 VNET_FOREACH(vnet_iter) {
1388                         CURVNET_SET(vnet_iter);
1389                         struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
1390                             INPLOOKUP_WLOCKPCB);
1391
1392                         while ((inp = inp_next(&inpi)) != NULL) {
1393                                 tp = intotcpcb(inp);
1394                                 if (tp == NULL || tp->t_fb != blk)
1395                                         continue;
1396                                 tcp_switch_back_to_default(tp);
1397                         }
1398                         CURVNET_RESTORE();
1399                 }
1400                 VNET_LIST_RUNLOCK();
1401
1402                 rw_wlock(&tcp_function_lock);
1403         }
1404         if (blk->tfb_refcnt) {
1405                 /* TCBs still attached. */
1406                 rw_wunlock(&tcp_function_lock);
1407                 return (EBUSY);
1408         }
1409         if (quiesce) {
1410                 /* Skip removal. */
1411                 rw_wunlock(&tcp_function_lock);
1412                 return (0);
1413         }
1414         /* Remove any function names that map to this function block. */
1415         while (find_tcp_fb_locked(blk, &f) != NULL) {
1416                 TAILQ_REMOVE(&t_functions, f, tf_next);
1417                 tcp_fb_cnt--;
1418                 f->tf_fb = NULL;
1419                 free(f, M_TCPFUNCTIONS);
1420         }
1421         rw_wunlock(&tcp_function_lock);
1422         return (0);
1423 }
1424
1425 static void
1426 tcp_drain(void)
1427 {
1428         struct epoch_tracker et;
1429         VNET_ITERATOR_DECL(vnet_iter);
1430
1431         if (!do_tcpdrain)
1432                 return;
1433
1434         NET_EPOCH_ENTER(et);
1435         VNET_LIST_RLOCK_NOSLEEP();
1436         VNET_FOREACH(vnet_iter) {
1437                 CURVNET_SET(vnet_iter);
1438                 struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
1439                     INPLOOKUP_WLOCKPCB);
1440                 struct inpcb *inpb;
1441                 struct tcpcb *tcpb;
1442
1443         /*
1444          * Walk the tcpbs, if existing, and flush the reassembly queue,
1445          * if there is one...
1446          * XXX: The "Net/3" implementation doesn't imply that the TCP
1447          *      reassembly queue should be flushed, but in a situation
1448          *      where we're really low on mbufs, this is potentially
1449          *      useful.
1450          */
1451                 while ((inpb = inp_next(&inpi)) != NULL) {
1452                         if ((tcpb = intotcpcb(inpb)) != NULL) {
1453                                 tcp_reass_flush(tcpb);
1454                                 tcp_clean_sackreport(tcpb);
1455 #ifdef TCP_BLACKBOX
1456                                 tcp_log_drain(tcpb);
1457 #endif
1458 #ifdef TCPPCAP
1459                                 if (tcp_pcap_aggressive_free) {
1460                                         /* Free the TCP PCAP queues. */
1461                                         tcp_pcap_drain(&(tcpb->t_inpkts));
1462                                         tcp_pcap_drain(&(tcpb->t_outpkts));
1463                                 }
1464 #endif
1465                         }
1466                 }
1467                 CURVNET_RESTORE();
1468         }
1469         VNET_LIST_RUNLOCK_NOSLEEP();
1470         NET_EPOCH_EXIT(et);
1471 }
1472
1473 static void
1474 tcp_vnet_init(void *arg __unused)
1475 {
1476
1477 #ifdef TCP_HHOOK
1478         if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN,
1479             &V_tcp_hhh[HHOOK_TCP_EST_IN], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
1480                 printf("%s: WARNING: unable to register helper hook\n", __func__);
1481         if (hhook_head_register(HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT,
1482             &V_tcp_hhh[HHOOK_TCP_EST_OUT], HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
1483                 printf("%s: WARNING: unable to register helper hook\n", __func__);
1484 #endif
1485 #ifdef STATS
1486         if (tcp_stats_init())
1487                 printf("%s: WARNING: unable to initialise TCP stats\n",
1488                     __func__);
1489 #endif
1490         in_pcbinfo_init(&V_tcbinfo, &tcpcbstor, tcp_tcbhashsize,
1491             tcp_tcbhashsize);
1492
1493         syncache_init();
1494         tcp_hc_init();
1495
1496         TUNABLE_INT_FETCH("net.inet.tcp.sack.enable", &V_tcp_do_sack);
1497         V_sack_hole_zone = uma_zcreate("sackhole", sizeof(struct sackhole),
1498             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
1499
1500         tcp_fastopen_init();
1501
1502         COUNTER_ARRAY_ALLOC(V_tcps_states, TCP_NSTATES, M_WAITOK);
1503         VNET_PCPUSTAT_ALLOC(tcpstat, M_WAITOK);
1504
1505         V_tcp_msl = TCPTV_MSL;
1506 }
1507 VNET_SYSINIT(tcp_vnet_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH,
1508     tcp_vnet_init, NULL);
1509
1510 static void
1511 tcp_init(void *arg __unused)
1512 {
1513         const char *tcbhash_tuneable;
1514         int hashsize;
1515
1516         tcp_reass_global_init();
1517
1518         /* XXX virtualize those below? */
1519         tcp_delacktime = TCPTV_DELACK;
1520         tcp_keepinit = TCPTV_KEEP_INIT;
1521         tcp_keepidle = TCPTV_KEEP_IDLE;
1522         tcp_keepintvl = TCPTV_KEEPINTVL;
1523         tcp_maxpersistidle = TCPTV_KEEP_IDLE;
1524         tcp_rexmit_initial = TCPTV_RTOBASE;
1525         if (tcp_rexmit_initial < 1)
1526                 tcp_rexmit_initial = 1;
1527         tcp_rexmit_min = TCPTV_MIN;
1528         if (tcp_rexmit_min < 1)
1529                 tcp_rexmit_min = 1;
1530         tcp_persmin = TCPTV_PERSMIN;
1531         tcp_persmax = TCPTV_PERSMAX;
1532         tcp_rexmit_slop = TCPTV_CPU_VAR;
1533         tcp_finwait2_timeout = TCPTV_FINWAIT2_TIMEOUT;
1534
1535         /* Setup the tcp function block list */
1536         TAILQ_INIT(&t_functions);
1537         rw_init(&tcp_function_lock, "tcp_func_lock");
1538         register_tcp_functions(&tcp_def_funcblk, M_WAITOK);
1539         sx_init(&tcpoudp_lock, "TCP over UDP configuration");
1540 #ifdef TCP_BLACKBOX
1541         /* Initialize the TCP logging data. */
1542         tcp_log_init();
1543 #endif
1544         arc4rand(&V_ts_offset_secret, sizeof(V_ts_offset_secret), 0);
1545
1546         if (tcp_soreceive_stream) {
1547 #ifdef INET
1548                 tcp_protosw.pr_soreceive = soreceive_stream;
1549 #endif
1550 #ifdef INET6
1551                 tcp6_protosw.pr_soreceive = soreceive_stream;
1552 #endif /* INET6 */
1553         }
1554
1555 #ifdef INET6
1556         max_protohdr_grow(sizeof(struct ip6_hdr) + sizeof(struct tcphdr));
1557 #else /* INET6 */
1558         max_protohdr_grow(sizeof(struct tcpiphdr));
1559 #endif /* INET6 */
1560
1561         ISN_LOCK_INIT();
1562         EVENTHANDLER_REGISTER(shutdown_pre_sync, tcp_fini, NULL,
1563                 SHUTDOWN_PRI_DEFAULT);
1564         EVENTHANDLER_REGISTER(vm_lowmem, tcp_drain, NULL, LOWMEM_PRI_DEFAULT);
1565         EVENTHANDLER_REGISTER(mbuf_lowmem, tcp_drain, NULL, LOWMEM_PRI_DEFAULT);
1566
1567         tcp_inp_lro_direct_queue = counter_u64_alloc(M_WAITOK);
1568         tcp_inp_lro_wokeup_queue = counter_u64_alloc(M_WAITOK);
1569         tcp_inp_lro_compressed = counter_u64_alloc(M_WAITOK);
1570         tcp_inp_lro_locks_taken = counter_u64_alloc(M_WAITOK);
1571         tcp_extra_mbuf = counter_u64_alloc(M_WAITOK);
1572         tcp_would_have_but = counter_u64_alloc(M_WAITOK);
1573         tcp_comp_total = counter_u64_alloc(M_WAITOK);
1574         tcp_uncomp_total = counter_u64_alloc(M_WAITOK);
1575         tcp_bad_csums = counter_u64_alloc(M_WAITOK);
1576         tcp_pacing_failures = counter_u64_alloc(M_WAITOK);
1577 #ifdef TCPPCAP
1578         tcp_pcap_init();
1579 #endif
1580
1581         hashsize = TCBHASHSIZE;
1582         tcbhash_tuneable = "net.inet.tcp.tcbhashsize";
1583         TUNABLE_INT_FETCH(tcbhash_tuneable, &hashsize);
1584         if (hashsize == 0) {
1585                 /*
1586                  * Auto tune the hash size based on maxsockets.
1587                  * A perfect hash would have a 1:1 mapping
1588                  * (hashsize = maxsockets) however it's been
1589                  * suggested that O(2) average is better.
1590                  */
1591                 hashsize = maketcp_hashsize(maxsockets / 4);
1592                 /*
1593                  * Our historical default is 512,
1594                  * do not autotune lower than this.
1595                  */
1596                 if (hashsize < 512)
1597                         hashsize = 512;
1598                 if (bootverbose)
1599                         printf("%s: %s auto tuned to %d\n", __func__,
1600                             tcbhash_tuneable, hashsize);
1601         }
1602         /*
1603          * We require a hashsize to be a power of two.
1604          * Previously if it was not a power of two we would just reset it
1605          * back to 512, which could be a nasty surprise if you did not notice
1606          * the error message.
1607          * Instead what we do is clip it to the closest power of two lower
1608          * than the specified hash value.
1609          */
1610         if (!powerof2(hashsize)) {
1611                 int oldhashsize = hashsize;
1612
1613                 hashsize = maketcp_hashsize(hashsize);
1614                 /* prevent absurdly low value */
1615                 if (hashsize < 16)
1616                         hashsize = 16;
1617                 printf("%s: WARNING: TCB hash size not a power of 2, "
1618                     "clipped from %d to %d.\n", __func__, oldhashsize,
1619                     hashsize);
1620         }
1621         tcp_tcbhashsize = hashsize;
1622
1623 #ifdef INET
1624         IPPROTO_REGISTER(IPPROTO_TCP, tcp_input, tcp_ctlinput);
1625 #endif
1626 #ifdef INET6
1627         IP6PROTO_REGISTER(IPPROTO_TCP, tcp6_input, tcp6_ctlinput);
1628 #endif
1629 }
1630 SYSINIT(tcp_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, tcp_init, NULL);
1631
1632 #ifdef VIMAGE
1633 static void
1634 tcp_destroy(void *unused __unused)
1635 {
1636         int n;
1637 #ifdef TCP_HHOOK
1638         int error;
1639 #endif
1640
1641         /*
1642          * All our processes are gone, all our sockets should be cleaned
1643          * up, which means, we should be past the tcp_discardcb() calls.
1644          * Sleep to let all tcpcb timers really disappear and cleanup.
1645          */
1646         for (;;) {
1647                 INP_INFO_WLOCK(&V_tcbinfo);
1648                 n = V_tcbinfo.ipi_count;
1649                 INP_INFO_WUNLOCK(&V_tcbinfo);
1650                 if (n == 0)
1651                         break;
1652                 pause("tcpdes", hz / 10);
1653         }
1654         tcp_hc_destroy();
1655         syncache_destroy();
1656         in_pcbinfo_destroy(&V_tcbinfo);
1657         /* tcp_discardcb() clears the sack_holes up. */
1658         uma_zdestroy(V_sack_hole_zone);
1659
1660         /*
1661          * Cannot free the zone until all tcpcbs are released as we attach
1662          * the allocations to them.
1663          */
1664         tcp_fastopen_destroy();
1665
1666         COUNTER_ARRAY_FREE(V_tcps_states, TCP_NSTATES);
1667         VNET_PCPUSTAT_FREE(tcpstat);
1668
1669 #ifdef TCP_HHOOK
1670         error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_IN]);
1671         if (error != 0) {
1672                 printf("%s: WARNING: unable to deregister helper hook "
1673                     "type=%d, id=%d: error %d returned\n", __func__,
1674                     HHOOK_TYPE_TCP, HHOOK_TCP_EST_IN, error);
1675         }
1676         error = hhook_head_deregister(V_tcp_hhh[HHOOK_TCP_EST_OUT]);
1677         if (error != 0) {
1678                 printf("%s: WARNING: unable to deregister helper hook "
1679                     "type=%d, id=%d: error %d returned\n", __func__,
1680                     HHOOK_TYPE_TCP, HHOOK_TCP_EST_OUT, error);
1681         }
1682 #endif
1683 }
1684 VNET_SYSUNINIT(tcp, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH, tcp_destroy, NULL);
1685 #endif
1686
1687 void
1688 tcp_fini(void *xtp)
1689 {
1690
1691 }
1692
1693 /*
1694  * Fill in the IP and TCP headers for an outgoing packet, given the tcpcb.
1695  * tcp_template used to store this data in mbufs, but we now recopy it out
1696  * of the tcpcb each time to conserve mbufs.
1697  */
1698 void
1699 tcpip_fillheaders(struct inpcb *inp, uint16_t port, void *ip_ptr, void *tcp_ptr)
1700 {
1701         struct tcphdr *th = (struct tcphdr *)tcp_ptr;
1702
1703         INP_WLOCK_ASSERT(inp);
1704
1705 #ifdef INET6
1706         if ((inp->inp_vflag & INP_IPV6) != 0) {
1707                 struct ip6_hdr *ip6;
1708
1709                 ip6 = (struct ip6_hdr *)ip_ptr;
1710                 ip6->ip6_flow = (ip6->ip6_flow & ~IPV6_FLOWINFO_MASK) |
1711                         (inp->inp_flow & IPV6_FLOWINFO_MASK);
1712                 ip6->ip6_vfc = (ip6->ip6_vfc & ~IPV6_VERSION_MASK) |
1713                         (IPV6_VERSION & IPV6_VERSION_MASK);
1714                 if (port == 0)
1715                         ip6->ip6_nxt = IPPROTO_TCP;
1716                 else
1717                         ip6->ip6_nxt = IPPROTO_UDP;
1718                 ip6->ip6_plen = htons(sizeof(struct tcphdr));
1719                 ip6->ip6_src = inp->in6p_laddr;
1720                 ip6->ip6_dst = inp->in6p_faddr;
1721         }
1722 #endif /* INET6 */
1723 #if defined(INET6) && defined(INET)
1724         else
1725 #endif
1726 #ifdef INET
1727         {
1728                 struct ip *ip;
1729
1730                 ip = (struct ip *)ip_ptr;
1731                 ip->ip_v = IPVERSION;
1732                 ip->ip_hl = 5;
1733                 ip->ip_tos = inp->inp_ip_tos;
1734                 ip->ip_len = 0;
1735                 ip->ip_id = 0;
1736                 ip->ip_off = 0;
1737                 ip->ip_ttl = inp->inp_ip_ttl;
1738                 ip->ip_sum = 0;
1739                 if (port == 0)
1740                         ip->ip_p = IPPROTO_TCP;
1741                 else
1742                         ip->ip_p = IPPROTO_UDP;
1743                 ip->ip_src = inp->inp_laddr;
1744                 ip->ip_dst = inp->inp_faddr;
1745         }
1746 #endif /* INET */
1747         th->th_sport = inp->inp_lport;
1748         th->th_dport = inp->inp_fport;
1749         th->th_seq = 0;
1750         th->th_ack = 0;
1751         th->th_off = 5;
1752         tcp_set_flags(th, 0);
1753         th->th_win = 0;
1754         th->th_urp = 0;
1755         th->th_sum = 0;         /* in_pseudo() is called later for ipv4 */
1756 }
1757
1758 /*
1759  * Create template to be used to send tcp packets on a connection.
1760  * Allocates an mbuf and fills in a skeletal tcp/ip header.  The only
1761  * use for this function is in keepalives, which use tcp_respond.
1762  */
1763 struct tcptemp *
1764 tcpip_maketemplate(struct inpcb *inp)
1765 {
1766         struct tcptemp *t;
1767
1768         t = malloc(sizeof(*t), M_TEMP, M_NOWAIT);
1769         if (t == NULL)
1770                 return (NULL);
1771         tcpip_fillheaders(inp, 0, (void *)&t->tt_ipgen, (void *)&t->tt_t);
1772         return (t);
1773 }
1774
1775 /*
1776  * Send a single message to the TCP at address specified by
1777  * the given TCP/IP header.  If m == NULL, then we make a copy
1778  * of the tcpiphdr at th and send directly to the addressed host.
1779  * This is used to force keep alive messages out using the TCP
1780  * template for a connection.  If flags are given then we send
1781  * a message back to the TCP which originated the segment th,
1782  * and discard the mbuf containing it and any other attached mbufs.
1783  *
1784  * In any case the ack and sequence number of the transmitted
1785  * segment are as specified by the parameters.
1786  *
1787  * NOTE: If m != NULL, then th must point to *inside* the mbuf.
1788  */
1789 void
1790 tcp_respond(struct tcpcb *tp, void *ipgen, struct tcphdr *th, struct mbuf *m,
1791     tcp_seq ack, tcp_seq seq, uint16_t flags)
1792 {
1793         struct tcpopt to;
1794         struct inpcb *inp;
1795         struct ip *ip;
1796         struct mbuf *optm;
1797         struct udphdr *uh = NULL;
1798         struct tcphdr *nth;
1799         struct tcp_log_buffer *lgb;
1800         u_char *optp;
1801 #ifdef INET6
1802         struct ip6_hdr *ip6;
1803         int isipv6;
1804 #endif /* INET6 */
1805         int optlen, tlen, win, ulen;
1806         int ect = 0;
1807         bool incl_opts;
1808         uint16_t port;
1809         int output_ret;
1810 #ifdef INVARIANTS
1811         int thflags = tcp_get_flags(th);
1812 #endif
1813
1814         KASSERT(tp != NULL || m != NULL, ("tcp_respond: tp and m both NULL"));
1815         NET_EPOCH_ASSERT();
1816
1817 #ifdef INET6
1818         isipv6 = ((struct ip *)ipgen)->ip_v == (IPV6_VERSION >> 4);
1819         ip6 = ipgen;
1820 #endif /* INET6 */
1821         ip = ipgen;
1822
1823         if (tp != NULL) {
1824                 inp = tptoinpcb(tp);
1825                 INP_LOCK_ASSERT(inp);
1826         } else
1827                 inp = NULL;
1828
1829         if (m != NULL) {
1830 #ifdef INET6
1831                 if (isipv6 && ip6 && (ip6->ip6_nxt == IPPROTO_UDP))
1832                         port = m->m_pkthdr.tcp_tun_port;
1833                 else
1834 #endif
1835                 if (ip && (ip->ip_p == IPPROTO_UDP))
1836                         port = m->m_pkthdr.tcp_tun_port;
1837                 else
1838                         port = 0;
1839         } else
1840                 port = tp->t_port;
1841
1842         incl_opts = false;
1843         win = 0;
1844         if (tp != NULL) {
1845                 if (!(flags & TH_RST)) {
1846                         win = sbspace(&inp->inp_socket->so_rcv);
1847                         if (win > TCP_MAXWIN << tp->rcv_scale)
1848                                 win = TCP_MAXWIN << tp->rcv_scale;
1849                 }
1850                 if ((tp->t_flags & TF_NOOPT) == 0)
1851                         incl_opts = true;
1852         }
1853         if (m == NULL) {
1854                 m = m_gethdr(M_NOWAIT, MT_DATA);
1855                 if (m == NULL)
1856                         return;
1857                 m->m_data += max_linkhdr;
1858 #ifdef INET6
1859                 if (isipv6) {
1860                         bcopy((caddr_t)ip6, mtod(m, caddr_t),
1861                               sizeof(struct ip6_hdr));
1862                         ip6 = mtod(m, struct ip6_hdr *);
1863                         nth = (struct tcphdr *)(ip6 + 1);
1864                         if (port) {
1865                                 /* Insert a UDP header */
1866                                 uh = (struct udphdr *)nth;
1867                                 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1868                                 uh->uh_dport = port;
1869                                 nth = (struct tcphdr *)(uh + 1);
1870                         }
1871                 } else
1872 #endif /* INET6 */
1873                 {
1874                         bcopy((caddr_t)ip, mtod(m, caddr_t), sizeof(struct ip));
1875                         ip = mtod(m, struct ip *);
1876                         nth = (struct tcphdr *)(ip + 1);
1877                         if (port) {
1878                                 /* Insert a UDP header */
1879                                 uh = (struct udphdr *)nth;
1880                                 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1881                                 uh->uh_dport = port;
1882                                 nth = (struct tcphdr *)(uh + 1);
1883                         }
1884                 }
1885                 bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
1886                 flags = TH_ACK;
1887         } else if ((!M_WRITABLE(m)) || (port != 0)) {
1888                 struct mbuf *n;
1889
1890                 /* Can't reuse 'm', allocate a new mbuf. */
1891                 n = m_gethdr(M_NOWAIT, MT_DATA);
1892                 if (n == NULL) {
1893                         m_freem(m);
1894                         return;
1895                 }
1896
1897                 if (!m_dup_pkthdr(n, m, M_NOWAIT)) {
1898                         m_freem(m);
1899                         m_freem(n);
1900                         return;
1901                 }
1902
1903                 n->m_data += max_linkhdr;
1904                 /* m_len is set later */
1905 #define xchg(a,b,type) { type t; t=a; a=b; b=t; }
1906 #ifdef INET6
1907                 if (isipv6) {
1908                         bcopy((caddr_t)ip6, mtod(n, caddr_t),
1909                               sizeof(struct ip6_hdr));
1910                         ip6 = mtod(n, struct ip6_hdr *);
1911                         xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
1912                         nth = (struct tcphdr *)(ip6 + 1);
1913                         if (port) {
1914                                 /* Insert a UDP header */
1915                                 uh = (struct udphdr *)nth;
1916                                 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1917                                 uh->uh_dport = port;
1918                                 nth = (struct tcphdr *)(uh + 1);
1919                         }
1920                 } else
1921 #endif /* INET6 */
1922                 {
1923                         bcopy((caddr_t)ip, mtod(n, caddr_t), sizeof(struct ip));
1924                         ip = mtod(n, struct ip *);
1925                         xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
1926                         nth = (struct tcphdr *)(ip + 1);
1927                         if (port) {
1928                                 /* Insert a UDP header */
1929                                 uh = (struct udphdr *)nth;
1930                                 uh->uh_sport = htons(V_tcp_udp_tunneling_port);
1931                                 uh->uh_dport = port;
1932                                 nth = (struct tcphdr *)(uh + 1);
1933                         }
1934                 }
1935                 bcopy((caddr_t)th, (caddr_t)nth, sizeof(struct tcphdr));
1936                 xchg(nth->th_dport, nth->th_sport, uint16_t);
1937                 th = nth;
1938                 m_freem(m);
1939                 m = n;
1940         } else {
1941                 /*
1942                  *  reuse the mbuf.
1943                  * XXX MRT We inherit the FIB, which is lucky.
1944                  */
1945                 m_freem(m->m_next);
1946                 m->m_next = NULL;
1947                 m->m_data = (caddr_t)ipgen;
1948                 /* clear any receive flags for proper bpf timestamping */
1949                 m->m_flags &= ~(M_TSTMP | M_TSTMP_LRO);
1950                 /* m_len is set later */
1951 #ifdef INET6
1952                 if (isipv6) {
1953                         xchg(ip6->ip6_dst, ip6->ip6_src, struct in6_addr);
1954                         nth = (struct tcphdr *)(ip6 + 1);
1955                 } else
1956 #endif /* INET6 */
1957                 {
1958                         xchg(ip->ip_dst.s_addr, ip->ip_src.s_addr, uint32_t);
1959                         nth = (struct tcphdr *)(ip + 1);
1960                 }
1961                 if (th != nth) {
1962                         /*
1963                          * this is usually a case when an extension header
1964                          * exists between the IPv6 header and the
1965                          * TCP header.
1966                          */
1967                         nth->th_sport = th->th_sport;
1968                         nth->th_dport = th->th_dport;
1969                 }
1970                 xchg(nth->th_dport, nth->th_sport, uint16_t);
1971 #undef xchg
1972         }
1973         tlen = 0;
1974 #ifdef INET6
1975         if (isipv6)
1976                 tlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
1977 #endif
1978 #if defined(INET) && defined(INET6)
1979         else
1980 #endif
1981 #ifdef INET
1982                 tlen = sizeof (struct tcpiphdr);
1983 #endif
1984         if (port)
1985                 tlen += sizeof (struct udphdr);
1986 #ifdef INVARIANTS
1987         m->m_len = 0;
1988         KASSERT(M_TRAILINGSPACE(m) >= tlen,
1989             ("Not enough trailing space for message (m=%p, need=%d, have=%ld)",
1990             m, tlen, (long)M_TRAILINGSPACE(m)));
1991 #endif
1992         m->m_len = tlen;
1993         to.to_flags = 0;
1994         if (incl_opts) {
1995                 ect = tcp_ecn_output_established(tp, &flags, 0, false);
1996                 /* Make sure we have room. */
1997                 if (M_TRAILINGSPACE(m) < TCP_MAXOLEN) {
1998                         m->m_next = m_get(M_NOWAIT, MT_DATA);
1999                         if (m->m_next) {
2000                                 optp = mtod(m->m_next, u_char *);
2001                                 optm = m->m_next;
2002                         } else
2003                                 incl_opts = false;
2004                 } else {
2005                         optp = (u_char *) (nth + 1);
2006                         optm = m;
2007                 }
2008         }
2009         if (incl_opts) {
2010                 /* Timestamps. */
2011                 if (tp->t_flags & TF_RCVD_TSTMP) {
2012                         to.to_tsval = tcp_ts_getticks() + tp->ts_offset;
2013                         to.to_tsecr = tp->ts_recent;
2014                         to.to_flags |= TOF_TS;
2015                 }
2016 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2017                 /* TCP-MD5 (RFC2385). */
2018                 if (tp->t_flags & TF_SIGNATURE)
2019                         to.to_flags |= TOF_SIGNATURE;
2020 #endif
2021                 /* Add the options. */
2022                 tlen += optlen = tcp_addoptions(&to, optp);
2023
2024                 /* Update m_len in the correct mbuf. */
2025                 optm->m_len += optlen;
2026         } else
2027                 optlen = 0;
2028 #ifdef INET6
2029         if (isipv6) {
2030                 if (uh) {
2031                         ulen = tlen - sizeof(struct ip6_hdr);
2032                         uh->uh_ulen = htons(ulen);
2033                 }
2034                 ip6->ip6_flow = htonl(ect << IPV6_FLOWLABEL_LEN);
2035                 ip6->ip6_vfc = IPV6_VERSION;
2036                 if (port)
2037                         ip6->ip6_nxt = IPPROTO_UDP;
2038                 else
2039                         ip6->ip6_nxt = IPPROTO_TCP;
2040                 ip6->ip6_plen = htons(tlen - sizeof(*ip6));
2041         }
2042 #endif
2043 #if defined(INET) && defined(INET6)
2044         else
2045 #endif
2046 #ifdef INET
2047         {
2048                 if (uh) {
2049                         ulen = tlen - sizeof(struct ip);
2050                         uh->uh_ulen = htons(ulen);
2051                 }
2052                 ip->ip_tos = ect;
2053                 ip->ip_len = htons(tlen);
2054                 ip->ip_ttl = V_ip_defttl;
2055                 if (port) {
2056                         ip->ip_p = IPPROTO_UDP;
2057                 } else {
2058                         ip->ip_p = IPPROTO_TCP;
2059                 }
2060                 if (V_path_mtu_discovery)
2061                         ip->ip_off |= htons(IP_DF);
2062         }
2063 #endif
2064         m->m_pkthdr.len = tlen;
2065         m->m_pkthdr.rcvif = NULL;
2066 #ifdef MAC
2067         if (inp != NULL) {
2068                 /*
2069                  * Packet is associated with a socket, so allow the
2070                  * label of the response to reflect the socket label.
2071                  */
2072                 INP_LOCK_ASSERT(inp);
2073                 mac_inpcb_create_mbuf(inp, m);
2074         } else {
2075                 /*
2076                  * Packet is not associated with a socket, so possibly
2077                  * update the label in place.
2078                  */
2079                 mac_netinet_tcp_reply(m);
2080         }
2081 #endif
2082         nth->th_seq = htonl(seq);
2083         nth->th_ack = htonl(ack);
2084         nth->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
2085         tcp_set_flags(nth, flags);
2086         if (tp && (flags & TH_RST)) {
2087                 /* Log the reset */
2088                 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
2089         }
2090         if (tp != NULL)
2091                 nth->th_win = htons((u_short) (win >> tp->rcv_scale));
2092         else
2093                 nth->th_win = htons((u_short)win);
2094         nth->th_urp = 0;
2095
2096 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
2097         if (to.to_flags & TOF_SIGNATURE) {
2098                 if (!TCPMD5_ENABLED() ||
2099                     TCPMD5_OUTPUT(m, nth, to.to_signature) != 0) {
2100                         m_freem(m);
2101                         return;
2102                 }
2103         }
2104 #endif
2105
2106 #ifdef INET6
2107         if (isipv6) {
2108                 if (port) {
2109                         m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
2110                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2111                         uh->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
2112                         nth->th_sum = 0;
2113                 } else {
2114                         m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
2115                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2116                         nth->th_sum = in6_cksum_pseudo(ip6,
2117                             tlen - sizeof(struct ip6_hdr), IPPROTO_TCP, 0);
2118                 }
2119                 ip6->ip6_hlim = in6_selecthlim(inp, NULL);
2120         }
2121 #endif /* INET6 */
2122 #if defined(INET6) && defined(INET)
2123         else
2124 #endif
2125 #ifdef INET
2126         {
2127                 if (port) {
2128                         uh->uh_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2129                             htons(ulen + IPPROTO_UDP));
2130                         m->m_pkthdr.csum_flags = CSUM_UDP;
2131                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
2132                         nth->th_sum = 0;
2133                 } else {
2134                         m->m_pkthdr.csum_flags = CSUM_TCP;
2135                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
2136                         nth->th_sum = in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
2137                             htons((u_short)(tlen - sizeof(struct ip) + ip->ip_p)));
2138                 }
2139         }
2140 #endif /* INET */
2141         TCP_PROBE3(debug__output, tp, th, m);
2142         if (flags & TH_RST)
2143                 TCP_PROBE5(accept__refused, NULL, NULL, m, tp, nth);
2144         lgb = NULL;
2145         if ((tp != NULL) && tcp_bblogging_on(tp)) {
2146                 if (INP_WLOCKED(inp)) {
2147                         union tcp_log_stackspecific log;
2148                         struct timeval tv;
2149
2150                         memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2151                         log.u_bbr.inhpts = tcp_in_hpts(tp);
2152                         log.u_bbr.flex8 = 4;
2153                         log.u_bbr.pkts_out = tp->t_maxseg;
2154                         log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2155                         log.u_bbr.delivered = 0;
2156                         lgb = tcp_log_event(tp, nth, NULL, NULL, TCP_LOG_OUT,
2157                             ERRNO_UNK, 0, &log, false, NULL, NULL, 0, &tv);
2158                 } else {
2159                         /*
2160                          * We can not log the packet, since we only own the
2161                          * read lock, but a write lock is needed. The read lock
2162                          * is not upgraded to a write lock, since only getting
2163                          * the read lock was done intentionally to improve the
2164                          * handling of SYN flooding attacks.
2165                          * This happens only for pure SYN segments received in
2166                          * the initial CLOSED state, or received in a more
2167                          * advanced state than listen and the UDP encapsulation
2168                          * port is unexpected.
2169                          * The incoming SYN segments do not really belong to
2170                          * the TCP connection and the handling does not change
2171                          * the state of the TCP connection. Therefore, the
2172                          * sending of the RST segments is not logged. Please
2173                          * note that also the incoming SYN segments are not
2174                          * logged.
2175                          *
2176                          * The following code ensures that the above description
2177                          * is and stays correct.
2178                          */
2179                         KASSERT((thflags & (TH_ACK|TH_SYN)) == TH_SYN &&
2180                             (tp->t_state == TCPS_CLOSED ||
2181                             (tp->t_state > TCPS_LISTEN && tp->t_port != port)),
2182                             ("%s: Logging of TCP segment with flags 0x%b and "
2183                             "UDP encapsulation port %u skipped in state %s",
2184                             __func__, thflags, PRINT_TH_FLAGS,
2185                             ntohs(port), tcpstates[tp->t_state]));
2186                 }
2187         }
2188
2189         if (flags & TH_ACK)
2190                 TCPSTAT_INC(tcps_sndacks);
2191         else if (flags & (TH_SYN|TH_FIN|TH_RST))
2192                 TCPSTAT_INC(tcps_sndctrl);
2193         TCPSTAT_INC(tcps_sndtotal);
2194
2195 #ifdef INET6
2196         if (isipv6) {
2197                 TCP_PROBE5(send, NULL, tp, ip6, tp, nth);
2198                 output_ret = ip6_output(m, NULL, NULL, 0, NULL, NULL, inp);
2199         }
2200 #endif /* INET6 */
2201 #if defined(INET) && defined(INET6)
2202         else
2203 #endif
2204 #ifdef INET
2205         {
2206                 TCP_PROBE5(send, NULL, tp, ip, tp, nth);
2207                 output_ret = ip_output(m, NULL, NULL, 0, NULL, inp);
2208         }
2209 #endif
2210         if (lgb != NULL)
2211                 lgb->tlb_errno = output_ret;
2212 }
2213
2214 /*
2215  * Create a new TCP control block, making an empty reassembly queue and hooking
2216  * it to the argument protocol control block.  The `inp' parameter must have
2217  * come from the zone allocator set up by tcpcbstor declaration.
2218  */
2219 struct tcpcb *
2220 tcp_newtcpcb(struct inpcb *inp)
2221 {
2222         struct tcpcb *tp = intotcpcb(inp);
2223 #ifdef INET6
2224         int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
2225 #endif /* INET6 */
2226
2227         /*
2228          * Historically allocation was done with M_ZERO.  There is a lot of
2229          * code that rely on that.  For now take safe approach and zero whole
2230          * tcpcb.  This definitely can be optimized.
2231          */
2232         bzero(&tp->t_start_zero, t_zero_size);
2233
2234         /* Initialise cc_var struct for this tcpcb. */
2235         tp->t_ccv.type = IPPROTO_TCP;
2236         tp->t_ccv.ccvc.tcp = tp;
2237         rw_rlock(&tcp_function_lock);
2238         tp->t_fb = V_tcp_func_set_ptr;
2239         refcount_acquire(&tp->t_fb->tfb_refcnt);
2240         rw_runlock(&tcp_function_lock);
2241         /*
2242          * Use the current system default CC algorithm.
2243          */
2244         cc_attach(tp, CC_DEFAULT_ALGO());
2245
2246         if (CC_ALGO(tp)->cb_init != NULL)
2247                 if (CC_ALGO(tp)->cb_init(&tp->t_ccv, NULL) > 0) {
2248                         cc_detach(tp);
2249                         if (tp->t_fb->tfb_tcp_fb_fini)
2250                                 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2251                         refcount_release(&tp->t_fb->tfb_refcnt);
2252                         return (NULL);
2253                 }
2254
2255 #ifdef TCP_HHOOK
2256         if (khelp_init_osd(HELPER_CLASS_TCP, &tp->t_osd)) {
2257                 if (tp->t_fb->tfb_tcp_fb_fini)
2258                         (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2259                 refcount_release(&tp->t_fb->tfb_refcnt);
2260                 return (NULL);
2261         }
2262 #endif
2263
2264         TAILQ_INIT(&tp->t_segq);
2265         STAILQ_INIT(&tp->t_inqueue);
2266         tp->t_maxseg =
2267 #ifdef INET6
2268                 isipv6 ? V_tcp_v6mssdflt :
2269 #endif /* INET6 */
2270                 V_tcp_mssdflt;
2271
2272         /* All mbuf queue/ack compress flags should be off */
2273         tcp_lro_features_off(tp);
2274
2275         callout_init_rw(&tp->t_callout, &inp->inp_lock, CALLOUT_RETURNUNLOCKED);
2276         for (int i = 0; i < TT_N; i++)
2277                 tp->t_timers[i] = SBT_MAX;
2278
2279         switch (V_tcp_do_rfc1323) {
2280                 case 0:
2281                         break;
2282                 default:
2283                 case 1:
2284                         tp->t_flags = (TF_REQ_SCALE|TF_REQ_TSTMP);
2285                         break;
2286                 case 2:
2287                         tp->t_flags = TF_REQ_SCALE;
2288                         break;
2289                 case 3:
2290                         tp->t_flags = TF_REQ_TSTMP;
2291                         break;
2292         }
2293         if (V_tcp_do_sack)
2294                 tp->t_flags |= TF_SACK_PERMIT;
2295         TAILQ_INIT(&tp->snd_holes);
2296
2297         /*
2298          * Init srtt to TCPTV_SRTTBASE (0), so we can tell that we have no
2299          * rtt estimate.  Set rttvar so that srtt + 4 * rttvar gives
2300          * reasonable initial retransmit time.
2301          */
2302         tp->t_srtt = TCPTV_SRTTBASE;
2303         tp->t_rttvar = ((tcp_rexmit_initial - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
2304         tp->t_rttmin = tcp_rexmit_min;
2305         tp->t_rxtcur = tcp_rexmit_initial;
2306         tp->snd_cwnd = TCP_MAXWIN << TCP_MAX_WINSHIFT;
2307         tp->snd_ssthresh = TCP_MAXWIN << TCP_MAX_WINSHIFT;
2308         tp->t_rcvtime = ticks;
2309         /* We always start with ticks granularity */
2310         tp->t_tmr_granularity = TCP_TMR_GRANULARITY_TICKS;
2311         /*
2312          * IPv4 TTL initialization is necessary for an IPv6 socket as well,
2313          * because the socket may be bound to an IPv6 wildcard address,
2314          * which may match an IPv4-mapped IPv6 address.
2315          */
2316         inp->inp_ip_ttl = V_ip_defttl;
2317 #ifdef TCPHPTS
2318         tcp_hpts_init(tp);
2319 #endif
2320 #ifdef TCPPCAP
2321         /*
2322          * Init the TCP PCAP queues.
2323          */
2324         tcp_pcap_tcpcb_init(tp);
2325 #endif
2326 #ifdef TCP_BLACKBOX
2327         /* Initialize the per-TCPCB log data. */
2328         tcp_log_tcpcbinit(tp);
2329 #endif
2330         tp->t_pacing_rate = -1;
2331         if (tp->t_fb->tfb_tcp_fb_init) {
2332                 if ((*tp->t_fb->tfb_tcp_fb_init)(tp, &tp->t_fb_ptr)) {
2333                         refcount_release(&tp->t_fb->tfb_refcnt);
2334                         return (NULL);
2335                 }
2336         }
2337 #ifdef STATS
2338         if (V_tcp_perconn_stats_enable == 1)
2339                 tp->t_stats = stats_blob_alloc(V_tcp_perconn_stats_dflt_tpl, 0);
2340 #endif
2341         if (V_tcp_do_lrd)
2342                 tp->t_flags |= TF_LRD;
2343
2344         return (tp);
2345 }
2346
2347 /*
2348  * Drop a TCP connection, reporting
2349  * the specified error.  If connection is synchronized,
2350  * then send a RST to peer.
2351  */
2352 struct tcpcb *
2353 tcp_drop(struct tcpcb *tp, int errno)
2354 {
2355         struct socket *so = tptosocket(tp);
2356
2357         NET_EPOCH_ASSERT();
2358         INP_WLOCK_ASSERT(tptoinpcb(tp));
2359
2360         if (TCPS_HAVERCVDSYN(tp->t_state)) {
2361                 tcp_state_change(tp, TCPS_CLOSED);
2362                 /* Don't use tcp_output() here due to possible recursion. */
2363                 (void)tcp_output_nodrop(tp);
2364                 TCPSTAT_INC(tcps_drops);
2365         } else
2366                 TCPSTAT_INC(tcps_conndrops);
2367         if (errno == ETIMEDOUT && tp->t_softerror)
2368                 errno = tp->t_softerror;
2369         so->so_error = errno;
2370         return (tcp_close(tp));
2371 }
2372
2373 void
2374 tcp_discardcb(struct tcpcb *tp)
2375 {
2376         struct inpcb *inp = tptoinpcb(tp);
2377         struct socket *so = tptosocket(tp);
2378         struct mbuf *m;
2379 #ifdef INET6
2380         bool isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
2381 #endif
2382
2383         INP_WLOCK_ASSERT(inp);
2384
2385         tcp_timer_stop(tp);
2386         if (tp->t_fb->tfb_tcp_timer_stop_all) {
2387                 tp->t_fb->tfb_tcp_timer_stop_all(tp);
2388         }
2389
2390         /* free the reassembly queue, if any */
2391         tcp_reass_flush(tp);
2392
2393 #ifdef TCP_OFFLOAD
2394         /* Disconnect offload device, if any. */
2395         if (tp->t_flags & TF_TOE)
2396                 tcp_offload_detach(tp);
2397 #endif
2398
2399         tcp_free_sackholes(tp);
2400
2401 #ifdef TCPPCAP
2402         /* Free the TCP PCAP queues. */
2403         tcp_pcap_drain(&(tp->t_inpkts));
2404         tcp_pcap_drain(&(tp->t_outpkts));
2405 #endif
2406
2407         /* Allow the CC algorithm to clean up after itself. */
2408         if (CC_ALGO(tp)->cb_destroy != NULL)
2409                 CC_ALGO(tp)->cb_destroy(&tp->t_ccv);
2410         CC_DATA(tp) = NULL;
2411         /* Detach from the CC algorithm */
2412         cc_detach(tp);
2413
2414 #ifdef TCP_HHOOK
2415         khelp_destroy_osd(&tp->t_osd);
2416 #endif
2417 #ifdef STATS
2418         stats_blob_destroy(tp->t_stats);
2419 #endif
2420
2421         CC_ALGO(tp) = NULL;
2422         if ((m = STAILQ_FIRST(&tp->t_inqueue)) != NULL) {
2423                 struct mbuf *prev;
2424
2425                 STAILQ_INIT(&tp->t_inqueue);
2426                 STAILQ_FOREACH_FROM_SAFE(m, &tp->t_inqueue, m_stailqpkt, prev)
2427                         m_freem(m);
2428         }
2429         TCPSTATES_DEC(tp->t_state);
2430
2431         if (tp->t_fb->tfb_tcp_fb_fini)
2432                 (*tp->t_fb->tfb_tcp_fb_fini)(tp, 1);
2433         MPASS(!tcp_in_hpts(tp));
2434 #ifdef TCP_BLACKBOX
2435         tcp_log_tcpcbfini(tp);
2436 #endif
2437
2438         /*
2439          * If we got enough samples through the srtt filter,
2440          * save the rtt and rttvar in the routing entry.
2441          * 'Enough' is arbitrarily defined as 4 rtt samples.
2442          * 4 samples is enough for the srtt filter to converge
2443          * to within enough % of the correct value; fewer samples
2444          * and we could save a bogus rtt. The danger is not high
2445          * as tcp quickly recovers from everything.
2446          * XXX: Works very well but needs some more statistics!
2447          *
2448          * XXXRRS: Updating must be after the stack fini() since
2449          * that may be converting some internal representation of
2450          * say srtt etc into the general one used by other stacks.
2451          * Lets also at least protect against the so being NULL
2452          * as RW stated below.
2453          */
2454         if ((tp->t_rttupdated >= 4) && (so != NULL)) {
2455                 struct hc_metrics_lite metrics;
2456                 uint32_t ssthresh;
2457
2458                 bzero(&metrics, sizeof(metrics));
2459                 /*
2460                  * Update the ssthresh always when the conditions below
2461                  * are satisfied. This gives us better new start value
2462                  * for the congestion avoidance for new connections.
2463                  * ssthresh is only set if packet loss occurred on a session.
2464                  *
2465                  * XXXRW: 'so' may be NULL here, and/or socket buffer may be
2466                  * being torn down.  Ideally this code would not use 'so'.
2467                  */
2468                 ssthresh = tp->snd_ssthresh;
2469                 if (ssthresh != 0 && ssthresh < so->so_snd.sb_hiwat / 2) {
2470                         /*
2471                          * convert the limit from user data bytes to
2472                          * packets then to packet data bytes.
2473                          */
2474                         ssthresh = (ssthresh + tp->t_maxseg / 2) / tp->t_maxseg;
2475                         if (ssthresh < 2)
2476                                 ssthresh = 2;
2477                         ssthresh *= (tp->t_maxseg +
2478 #ifdef INET6
2479                             (isipv6 ? sizeof (struct ip6_hdr) +
2480                             sizeof (struct tcphdr) :
2481 #endif
2482                             sizeof (struct tcpiphdr)
2483 #ifdef INET6
2484                             )
2485 #endif
2486                             );
2487                 } else
2488                         ssthresh = 0;
2489                 metrics.rmx_ssthresh = ssthresh;
2490
2491                 metrics.rmx_rtt = tp->t_srtt;
2492                 metrics.rmx_rttvar = tp->t_rttvar;
2493                 metrics.rmx_cwnd = tp->snd_cwnd;
2494                 metrics.rmx_sendpipe = 0;
2495                 metrics.rmx_recvpipe = 0;
2496
2497                 tcp_hc_update(&inp->inp_inc, &metrics);
2498         }
2499
2500         refcount_release(&tp->t_fb->tfb_refcnt);
2501 }
2502
2503 /*
2504  * Attempt to close a TCP control block, marking it as dropped, and freeing
2505  * the socket if we hold the only reference.
2506  */
2507 struct tcpcb *
2508 tcp_close(struct tcpcb *tp)
2509 {
2510         struct inpcb *inp = tptoinpcb(tp);
2511         struct socket *so = tptosocket(tp);
2512
2513         INP_WLOCK_ASSERT(inp);
2514
2515 #ifdef TCP_OFFLOAD
2516         if (tp->t_state == TCPS_LISTEN)
2517                 tcp_offload_listen_stop(tp);
2518 #endif
2519         /*
2520          * This releases the TFO pending counter resource for TFO listen
2521          * sockets as well as passively-created TFO sockets that transition
2522          * from SYN_RECEIVED to CLOSED.
2523          */
2524         if (tp->t_tfo_pending) {
2525                 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
2526                 tp->t_tfo_pending = NULL;
2527         }
2528 #ifdef TCPHPTS
2529         tcp_hpts_remove(tp);
2530 #endif
2531         in_pcbdrop(inp);
2532         TCPSTAT_INC(tcps_closed);
2533         if (tp->t_state != TCPS_CLOSED)
2534                 tcp_state_change(tp, TCPS_CLOSED);
2535         KASSERT(inp->inp_socket != NULL, ("tcp_close: inp_socket NULL"));
2536         soisdisconnected(so);
2537         if (inp->inp_flags & INP_SOCKREF) {
2538                 inp->inp_flags &= ~INP_SOCKREF;
2539                 INP_WUNLOCK(inp);
2540                 sorele(so);
2541                 return (NULL);
2542         }
2543         return (tp);
2544 }
2545
2546 /*
2547  * Notify a tcp user of an asynchronous error;
2548  * store error as soft error, but wake up user
2549  * (for now, won't do anything until can select for soft error).
2550  *
2551  * Do not wake up user since there currently is no mechanism for
2552  * reporting soft errors (yet - a kqueue filter may be added).
2553  */
2554 static struct inpcb *
2555 tcp_notify(struct inpcb *inp, int error)
2556 {
2557         struct tcpcb *tp;
2558
2559         INP_WLOCK_ASSERT(inp);
2560
2561         tp = intotcpcb(inp);
2562         KASSERT(tp != NULL, ("tcp_notify: tp == NULL"));
2563
2564         /*
2565          * Ignore some errors if we are hooked up.
2566          * If connection hasn't completed, has retransmitted several times,
2567          * and receives a second error, give up now.  This is better
2568          * than waiting a long time to establish a connection that
2569          * can never complete.
2570          */
2571         if (tp->t_state == TCPS_ESTABLISHED &&
2572             (error == EHOSTUNREACH || error == ENETUNREACH ||
2573              error == EHOSTDOWN)) {
2574                 if (inp->inp_route.ro_nh) {
2575                         NH_FREE(inp->inp_route.ro_nh);
2576                         inp->inp_route.ro_nh = (struct nhop_object *)NULL;
2577                 }
2578                 return (inp);
2579         } else if (tp->t_state < TCPS_ESTABLISHED && tp->t_rxtshift > 3 &&
2580             tp->t_softerror) {
2581                 tp = tcp_drop(tp, error);
2582                 if (tp != NULL)
2583                         return (inp);
2584                 else
2585                         return (NULL);
2586         } else {
2587                 tp->t_softerror = error;
2588                 return (inp);
2589         }
2590 #if 0
2591         wakeup( &so->so_timeo);
2592         sorwakeup(so);
2593         sowwakeup(so);
2594 #endif
2595 }
2596
2597 static int
2598 tcp_pcblist(SYSCTL_HANDLER_ARGS)
2599 {
2600         struct inpcb_iterator inpi = INP_ALL_ITERATOR(&V_tcbinfo,
2601             INPLOOKUP_RLOCKPCB);
2602         struct xinpgen xig;
2603         struct inpcb *inp;
2604         int error;
2605
2606         if (req->newptr != NULL)
2607                 return (EPERM);
2608
2609         if (req->oldptr == NULL) {
2610                 int n;
2611
2612                 n = V_tcbinfo.ipi_count +
2613                     counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2614                 n += imax(n / 8, 10);
2615                 req->oldidx = 2 * (sizeof xig) + n * sizeof(struct xtcpcb);
2616                 return (0);
2617         }
2618
2619         if ((error = sysctl_wire_old_buffer(req, 0)) != 0)
2620                 return (error);
2621
2622         bzero(&xig, sizeof(xig));
2623         xig.xig_len = sizeof xig;
2624         xig.xig_count = V_tcbinfo.ipi_count +
2625             counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2626         xig.xig_gen = V_tcbinfo.ipi_gencnt;
2627         xig.xig_sogen = so_gencnt;
2628         error = SYSCTL_OUT(req, &xig, sizeof xig);
2629         if (error)
2630                 return (error);
2631
2632         error = syncache_pcblist(req);
2633         if (error)
2634                 return (error);
2635
2636         while ((inp = inp_next(&inpi)) != NULL) {
2637                 if (inp->inp_gencnt <= xig.xig_gen &&
2638                     cr_canseeinpcb(req->td->td_ucred, inp) == 0) {
2639                         struct xtcpcb xt;
2640
2641                         tcp_inptoxtp(inp, &xt);
2642                         error = SYSCTL_OUT(req, &xt, sizeof xt);
2643                         if (error) {
2644                                 INP_RUNLOCK(inp);
2645                                 break;
2646                         } else
2647                                 continue;
2648                 }
2649         }
2650
2651         if (!error) {
2652                 /*
2653                  * Give the user an updated idea of our state.
2654                  * If the generation differs from what we told
2655                  * her before, she knows that something happened
2656                  * while we were processing this request, and it
2657                  * might be necessary to retry.
2658                  */
2659                 xig.xig_gen = V_tcbinfo.ipi_gencnt;
2660                 xig.xig_sogen = so_gencnt;
2661                 xig.xig_count = V_tcbinfo.ipi_count +
2662                     counter_u64_fetch(V_tcps_states[TCPS_SYN_RECEIVED]);
2663                 error = SYSCTL_OUT(req, &xig, sizeof xig);
2664         }
2665
2666         return (error);
2667 }
2668
2669 SYSCTL_PROC(_net_inet_tcp, TCPCTL_PCBLIST, pcblist,
2670     CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_NEEDGIANT,
2671     NULL, 0, tcp_pcblist, "S,xtcpcb",
2672     "List of active TCP connections");
2673
2674 #ifdef INET
2675 static int
2676 tcp_getcred(SYSCTL_HANDLER_ARGS)
2677 {
2678         struct xucred xuc;
2679         struct sockaddr_in addrs[2];
2680         struct epoch_tracker et;
2681         struct inpcb *inp;
2682         int error;
2683
2684         error = priv_check(req->td, PRIV_NETINET_GETCRED);
2685         if (error)
2686                 return (error);
2687         error = SYSCTL_IN(req, addrs, sizeof(addrs));
2688         if (error)
2689                 return (error);
2690         NET_EPOCH_ENTER(et);
2691         inp = in_pcblookup(&V_tcbinfo, addrs[1].sin_addr, addrs[1].sin_port,
2692             addrs[0].sin_addr, addrs[0].sin_port, INPLOOKUP_RLOCKPCB, NULL);
2693         NET_EPOCH_EXIT(et);
2694         if (inp != NULL) {
2695                 if (error == 0)
2696                         error = cr_canseeinpcb(req->td->td_ucred, inp);
2697                 if (error == 0)
2698                         cru2x(inp->inp_cred, &xuc);
2699                 INP_RUNLOCK(inp);
2700         } else
2701                 error = ENOENT;
2702         if (error == 0)
2703                 error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
2704         return (error);
2705 }
2706
2707 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, getcred,
2708     CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_NEEDGIANT,
2709     0, 0, tcp_getcred, "S,xucred",
2710     "Get the xucred of a TCP connection");
2711 #endif /* INET */
2712
2713 #ifdef INET6
2714 static int
2715 tcp6_getcred(SYSCTL_HANDLER_ARGS)
2716 {
2717         struct epoch_tracker et;
2718         struct xucred xuc;
2719         struct sockaddr_in6 addrs[2];
2720         struct inpcb *inp;
2721         int error;
2722 #ifdef INET
2723         int mapped = 0;
2724 #endif
2725
2726         error = priv_check(req->td, PRIV_NETINET_GETCRED);
2727         if (error)
2728                 return (error);
2729         error = SYSCTL_IN(req, addrs, sizeof(addrs));
2730         if (error)
2731                 return (error);
2732         if ((error = sa6_embedscope(&addrs[0], V_ip6_use_defzone)) != 0 ||
2733             (error = sa6_embedscope(&addrs[1], V_ip6_use_defzone)) != 0) {
2734                 return (error);
2735         }
2736         if (IN6_IS_ADDR_V4MAPPED(&addrs[0].sin6_addr)) {
2737 #ifdef INET
2738                 if (IN6_IS_ADDR_V4MAPPED(&addrs[1].sin6_addr))
2739                         mapped = 1;
2740                 else
2741 #endif
2742                         return (EINVAL);
2743         }
2744
2745         NET_EPOCH_ENTER(et);
2746 #ifdef INET
2747         if (mapped == 1)
2748                 inp = in_pcblookup(&V_tcbinfo,
2749                         *(struct in_addr *)&addrs[1].sin6_addr.s6_addr[12],
2750                         addrs[1].sin6_port,
2751                         *(struct in_addr *)&addrs[0].sin6_addr.s6_addr[12],
2752                         addrs[0].sin6_port, INPLOOKUP_RLOCKPCB, NULL);
2753         else
2754 #endif
2755                 inp = in6_pcblookup(&V_tcbinfo,
2756                         &addrs[1].sin6_addr, addrs[1].sin6_port,
2757                         &addrs[0].sin6_addr, addrs[0].sin6_port,
2758                         INPLOOKUP_RLOCKPCB, NULL);
2759         NET_EPOCH_EXIT(et);
2760         if (inp != NULL) {
2761                 if (error == 0)
2762                         error = cr_canseeinpcb(req->td->td_ucred, inp);
2763                 if (error == 0)
2764                         cru2x(inp->inp_cred, &xuc);
2765                 INP_RUNLOCK(inp);
2766         } else
2767                 error = ENOENT;
2768         if (error == 0)
2769                 error = SYSCTL_OUT(req, &xuc, sizeof(struct xucred));
2770         return (error);
2771 }
2772
2773 SYSCTL_PROC(_net_inet6_tcp6, OID_AUTO, getcred,
2774     CTLTYPE_OPAQUE | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_NEEDGIANT,
2775     0, 0, tcp6_getcred, "S,xucred",
2776     "Get the xucred of a TCP6 connection");
2777 #endif /* INET6 */
2778
2779 #ifdef INET
2780 /* Path MTU to try next when a fragmentation-needed message is received. */
2781 static inline int
2782 tcp_next_pmtu(const struct icmp *icp, const struct ip *ip)
2783 {
2784         int mtu = ntohs(icp->icmp_nextmtu);
2785
2786         /* If no alternative MTU was proposed, try the next smaller one. */
2787         if (!mtu)
2788                 mtu = ip_next_mtu(ntohs(ip->ip_len), 1);
2789         if (mtu < V_tcp_minmss + sizeof(struct tcpiphdr))
2790                 mtu = V_tcp_minmss + sizeof(struct tcpiphdr);
2791
2792         return (mtu);
2793 }
2794
2795 static void
2796 tcp_ctlinput_with_port(struct icmp *icp, uint16_t port)
2797 {
2798         struct ip *ip;
2799         struct tcphdr *th;
2800         struct inpcb *inp;
2801         struct tcpcb *tp;
2802         struct inpcb *(*notify)(struct inpcb *, int);
2803         struct in_conninfo inc;
2804         tcp_seq icmp_tcp_seq;
2805         int errno, mtu;
2806
2807         errno = icmp_errmap(icp);
2808         switch (errno) {
2809         case 0:
2810                 return;
2811         case EMSGSIZE:
2812                 notify = tcp_mtudisc_notify;
2813                 break;
2814         case ECONNREFUSED:
2815                 if (V_icmp_may_rst)
2816                         notify = tcp_drop_syn_sent;
2817                 else
2818                         notify = tcp_notify;
2819                 break;
2820         case EHOSTUNREACH:
2821                 if (V_icmp_may_rst && icp->icmp_type == ICMP_TIMXCEED)
2822                         notify = tcp_drop_syn_sent;
2823                 else
2824                         notify = tcp_notify;
2825                 break;
2826         default:
2827                 notify = tcp_notify;
2828         }
2829
2830         ip = &icp->icmp_ip;
2831         th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
2832         icmp_tcp_seq = th->th_seq;
2833         inp = in_pcblookup(&V_tcbinfo, ip->ip_dst, th->th_dport, ip->ip_src,
2834             th->th_sport, INPLOOKUP_WLOCKPCB, NULL);
2835         if (inp != NULL)  {
2836                 tp = intotcpcb(inp);
2837 #ifdef TCP_OFFLOAD
2838                 if (tp->t_flags & TF_TOE && errno == EMSGSIZE) {
2839                         /*
2840                          * MTU discovery for offloaded connections.  Let
2841                          * the TOE driver verify seq# and process it.
2842                          */
2843                         mtu = tcp_next_pmtu(icp, ip);
2844                         tcp_offload_pmtu_update(tp, icmp_tcp_seq, mtu);
2845                         goto out;
2846                 }
2847 #endif
2848                 if (tp->t_port != port)
2849                         goto out;
2850                 if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
2851                     SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
2852                         if (errno == EMSGSIZE) {
2853                                 /*
2854                                  * MTU discovery: we got a needfrag and
2855                                  * will potentially try a lower MTU.
2856                                  */
2857                                 mtu = tcp_next_pmtu(icp, ip);
2858
2859                                 /*
2860                                  * Only process the offered MTU if it
2861                                  * is smaller than the current one.
2862                                  */
2863                                 if (mtu < tp->t_maxseg +
2864                                     sizeof(struct tcpiphdr)) {
2865                                         bzero(&inc, sizeof(inc));
2866                                         inc.inc_faddr = ip->ip_dst;
2867                                         inc.inc_fibnum =
2868                                             inp->inp_inc.inc_fibnum;
2869                                         tcp_hc_updatemtu(&inc, mtu);
2870                                         inp = tcp_mtudisc(inp, mtu);
2871                                 }
2872                         } else
2873                                 inp = (*notify)(inp, errno);
2874                 }
2875         } else {
2876                 bzero(&inc, sizeof(inc));
2877                 inc.inc_fport = th->th_dport;
2878                 inc.inc_lport = th->th_sport;
2879                 inc.inc_faddr = ip->ip_dst;
2880                 inc.inc_laddr = ip->ip_src;
2881                 syncache_unreach(&inc, icmp_tcp_seq, port);
2882         }
2883 out:
2884         if (inp != NULL)
2885                 INP_WUNLOCK(inp);
2886 }
2887
2888 static void
2889 tcp_ctlinput(struct icmp *icmp)
2890 {
2891         tcp_ctlinput_with_port(icmp, htons(0));
2892 }
2893
2894 static void
2895 tcp_ctlinput_viaudp(udp_tun_icmp_param_t param)
2896 {
2897         /* Its a tunneled TCP over UDP icmp */
2898         struct icmp *icmp = param.icmp;
2899         struct ip *outer_ip, *inner_ip;
2900         struct udphdr *udp;
2901         struct tcphdr *th, ttemp;
2902         int i_hlen, o_len;
2903         uint16_t port;
2904
2905         outer_ip = (struct ip *)((caddr_t)icmp - sizeof(struct ip));
2906         inner_ip = &icmp->icmp_ip;
2907         i_hlen = inner_ip->ip_hl << 2;
2908         o_len = ntohs(outer_ip->ip_len);
2909         if (o_len <
2910             (sizeof(struct ip) + 8 + i_hlen + sizeof(struct udphdr) + offsetof(struct tcphdr, th_ack))) {
2911                 /* Not enough data present */
2912                 return;
2913         }
2914         /* Ok lets strip out the inner udphdr header by copying up on top of it the tcp hdr */
2915         udp = (struct udphdr *)(((caddr_t)inner_ip) + i_hlen);
2916         if (ntohs(udp->uh_sport) != V_tcp_udp_tunneling_port) {
2917                 return;
2918         }
2919         port = udp->uh_dport;
2920         th = (struct tcphdr *)(udp + 1);
2921         memcpy(&ttemp, th, sizeof(struct tcphdr));
2922         memcpy(udp, &ttemp, sizeof(struct tcphdr));
2923         /* Now adjust down the size of the outer IP header */
2924         o_len -= sizeof(struct udphdr);
2925         outer_ip->ip_len = htons(o_len);
2926         /* Now call in to the normal handling code */
2927         tcp_ctlinput_with_port(icmp, port);
2928 }
2929 #endif /* INET */
2930
2931 #ifdef INET6
2932 static inline int
2933 tcp6_next_pmtu(const struct icmp6_hdr *icmp6)
2934 {
2935         int mtu = ntohl(icmp6->icmp6_mtu);
2936
2937         /*
2938          * If no alternative MTU was proposed, or the proposed MTU was too
2939          * small, set to the min.
2940          */
2941         if (mtu < IPV6_MMTU)
2942                 mtu = IPV6_MMTU - 8;    /* XXXNP: what is the adjustment for? */
2943         return (mtu);
2944 }
2945
2946 static void
2947 tcp6_ctlinput_with_port(struct ip6ctlparam *ip6cp, uint16_t port)
2948 {
2949         struct in6_addr *dst;
2950         struct inpcb *(*notify)(struct inpcb *, int);
2951         struct ip6_hdr *ip6;
2952         struct mbuf *m;
2953         struct inpcb *inp;
2954         struct tcpcb *tp;
2955         struct icmp6_hdr *icmp6;
2956         struct in_conninfo inc;
2957         struct tcp_ports {
2958                 uint16_t th_sport;
2959                 uint16_t th_dport;
2960         } t_ports;
2961         tcp_seq icmp_tcp_seq;
2962         unsigned int mtu;
2963         unsigned int off;
2964         int errno;
2965
2966         icmp6 = ip6cp->ip6c_icmp6;
2967         m = ip6cp->ip6c_m;
2968         ip6 = ip6cp->ip6c_ip6;
2969         off = ip6cp->ip6c_off;
2970         dst = &ip6cp->ip6c_finaldst->sin6_addr;
2971
2972         errno = icmp6_errmap(icmp6);
2973         switch (errno) {
2974         case 0:
2975                 return;
2976         case EMSGSIZE:
2977                 notify = tcp_mtudisc_notify;
2978                 break;
2979         case ECONNREFUSED:
2980                 if (V_icmp_may_rst)
2981                         notify = tcp_drop_syn_sent;
2982                 else
2983                         notify = tcp_notify;
2984                 break;
2985         case EHOSTUNREACH:
2986                 /*
2987                  * There are only four ICMPs that may reset connection:
2988                  * - administratively prohibited
2989                  * - port unreachable
2990                  * - time exceeded in transit
2991                  * - unknown next header
2992                  */
2993                 if (V_icmp_may_rst &&
2994                     ((icmp6->icmp6_type == ICMP6_DST_UNREACH &&
2995                      (icmp6->icmp6_code == ICMP6_DST_UNREACH_ADMIN ||
2996                       icmp6->icmp6_code == ICMP6_DST_UNREACH_NOPORT)) ||
2997                     (icmp6->icmp6_type == ICMP6_TIME_EXCEEDED &&
2998                       icmp6->icmp6_code == ICMP6_TIME_EXCEED_TRANSIT) ||
2999                     (icmp6->icmp6_type == ICMP6_PARAM_PROB &&
3000                       icmp6->icmp6_code == ICMP6_PARAMPROB_NEXTHEADER)))
3001                         notify = tcp_drop_syn_sent;
3002                 else
3003                         notify = tcp_notify;
3004                 break;
3005         default:
3006                 notify = tcp_notify;
3007         }
3008
3009         /* Check if we can safely get the ports from the tcp hdr */
3010         if (m == NULL ||
3011             (m->m_pkthdr.len <
3012                 (int32_t) (off + sizeof(struct tcp_ports)))) {
3013                 return;
3014         }
3015         bzero(&t_ports, sizeof(struct tcp_ports));
3016         m_copydata(m, off, sizeof(struct tcp_ports), (caddr_t)&t_ports);
3017         inp = in6_pcblookup(&V_tcbinfo, &ip6->ip6_dst, t_ports.th_dport,
3018             &ip6->ip6_src, t_ports.th_sport, INPLOOKUP_WLOCKPCB, NULL);
3019         off += sizeof(struct tcp_ports);
3020         if (m->m_pkthdr.len < (int32_t) (off + sizeof(tcp_seq))) {
3021                 goto out;
3022         }
3023         m_copydata(m, off, sizeof(tcp_seq), (caddr_t)&icmp_tcp_seq);
3024         if (inp != NULL)  {
3025                 tp = intotcpcb(inp);
3026 #ifdef TCP_OFFLOAD
3027                 if (tp->t_flags & TF_TOE && errno == EMSGSIZE) {
3028                         /* MTU discovery for offloaded connections. */
3029                         mtu = tcp6_next_pmtu(icmp6);
3030                         tcp_offload_pmtu_update(tp, icmp_tcp_seq, mtu);
3031                         goto out;
3032                 }
3033 #endif
3034                 if (tp->t_port != port)
3035                         goto out;
3036                 if (SEQ_GEQ(ntohl(icmp_tcp_seq), tp->snd_una) &&
3037                     SEQ_LT(ntohl(icmp_tcp_seq), tp->snd_max)) {
3038                         if (errno == EMSGSIZE) {
3039                                 /*
3040                                  * MTU discovery:
3041                                  * If we got a needfrag set the MTU
3042                                  * in the route to the suggested new
3043                                  * value (if given) and then notify.
3044                                  */
3045                                 mtu = tcp6_next_pmtu(icmp6);
3046
3047                                 bzero(&inc, sizeof(inc));
3048                                 inc.inc_fibnum = M_GETFIB(m);
3049                                 inc.inc_flags |= INC_ISIPV6;
3050                                 inc.inc6_faddr = *dst;
3051                                 if (in6_setscope(&inc.inc6_faddr,
3052                                         m->m_pkthdr.rcvif, NULL))
3053                                         goto out;
3054                                 /*
3055                                  * Only process the offered MTU if it
3056                                  * is smaller than the current one.
3057                                  */
3058                                 if (mtu < tp->t_maxseg +
3059                                     sizeof (struct tcphdr) +
3060                                     sizeof (struct ip6_hdr)) {
3061                                         tcp_hc_updatemtu(&inc, mtu);
3062                                         tcp_mtudisc(inp, mtu);
3063                                         ICMP6STAT_INC(icp6s_pmtuchg);
3064                                 }
3065                         } else
3066                                 inp = (*notify)(inp, errno);
3067                 }
3068         } else {
3069                 bzero(&inc, sizeof(inc));
3070                 inc.inc_fibnum = M_GETFIB(m);
3071                 inc.inc_flags |= INC_ISIPV6;
3072                 inc.inc_fport = t_ports.th_dport;
3073                 inc.inc_lport = t_ports.th_sport;
3074                 inc.inc6_faddr = *dst;
3075                 inc.inc6_laddr = ip6->ip6_src;
3076                 syncache_unreach(&inc, icmp_tcp_seq, port);
3077         }
3078 out:
3079         if (inp != NULL)
3080                 INP_WUNLOCK(inp);
3081 }
3082
3083 static void
3084 tcp6_ctlinput(struct ip6ctlparam *ctl)
3085 {
3086         tcp6_ctlinput_with_port(ctl, htons(0));
3087 }
3088
3089 static void
3090 tcp6_ctlinput_viaudp(udp_tun_icmp_param_t param)
3091 {
3092         struct ip6ctlparam *ip6cp = param.ip6cp;
3093         struct mbuf *m;
3094         struct udphdr *udp;
3095         uint16_t port;
3096
3097         m = m_pulldown(ip6cp->ip6c_m, ip6cp->ip6c_off, sizeof(struct udphdr), NULL);
3098         if (m == NULL) {
3099                 return;
3100         }
3101         udp = mtod(m, struct udphdr *);
3102         if (ntohs(udp->uh_sport) != V_tcp_udp_tunneling_port) {
3103                 return;
3104         }
3105         port = udp->uh_dport;
3106         m_adj(m, sizeof(struct udphdr));
3107         if ((m->m_flags & M_PKTHDR) == 0) {
3108                 ip6cp->ip6c_m->m_pkthdr.len -= sizeof(struct udphdr);
3109         }
3110         /* Now call in to the normal handling code */
3111         tcp6_ctlinput_with_port(ip6cp, port);
3112 }
3113
3114 #endif /* INET6 */
3115
3116 static uint32_t
3117 tcp_keyed_hash(struct in_conninfo *inc, u_char *key, u_int len)
3118 {
3119         SIPHASH_CTX ctx;
3120         uint32_t hash[2];
3121
3122         KASSERT(len >= SIPHASH_KEY_LENGTH,
3123             ("%s: keylen %u too short ", __func__, len));
3124         SipHash24_Init(&ctx);
3125         SipHash_SetKey(&ctx, (uint8_t *)key);
3126         SipHash_Update(&ctx, &inc->inc_fport, sizeof(uint16_t));
3127         SipHash_Update(&ctx, &inc->inc_lport, sizeof(uint16_t));
3128         switch (inc->inc_flags & INC_ISIPV6) {
3129 #ifdef INET
3130         case 0:
3131                 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(struct in_addr));
3132                 SipHash_Update(&ctx, &inc->inc_laddr, sizeof(struct in_addr));
3133                 break;
3134 #endif
3135 #ifdef INET6
3136         case INC_ISIPV6:
3137                 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(struct in6_addr));
3138                 SipHash_Update(&ctx, &inc->inc6_laddr, sizeof(struct in6_addr));
3139                 break;
3140 #endif
3141         }
3142         SipHash_Final((uint8_t *)hash, &ctx);
3143
3144         return (hash[0] ^ hash[1]);
3145 }
3146
3147 uint32_t
3148 tcp_new_ts_offset(struct in_conninfo *inc)
3149 {
3150         struct in_conninfo inc_store, *local_inc;
3151
3152         if (!V_tcp_ts_offset_per_conn) {
3153                 memcpy(&inc_store, inc, sizeof(struct in_conninfo));
3154                 inc_store.inc_lport = 0;
3155                 inc_store.inc_fport = 0;
3156                 local_inc = &inc_store;
3157         } else {
3158                 local_inc = inc;
3159         }
3160         return (tcp_keyed_hash(local_inc, V_ts_offset_secret,
3161             sizeof(V_ts_offset_secret)));
3162 }
3163
3164 /*
3165  * Following is where TCP initial sequence number generation occurs.
3166  *
3167  * There are two places where we must use initial sequence numbers:
3168  * 1.  In SYN-ACK packets.
3169  * 2.  In SYN packets.
3170  *
3171  * All ISNs for SYN-ACK packets are generated by the syncache.  See
3172  * tcp_syncache.c for details.
3173  *
3174  * The ISNs in SYN packets must be monotonic; TIME_WAIT recycling
3175  * depends on this property.  In addition, these ISNs should be
3176  * unguessable so as to prevent connection hijacking.  To satisfy
3177  * the requirements of this situation, the algorithm outlined in
3178  * RFC 1948 is used, with only small modifications.
3179  *
3180  * Implementation details:
3181  *
3182  * Time is based off the system timer, and is corrected so that it
3183  * increases by one megabyte per second.  This allows for proper
3184  * recycling on high speed LANs while still leaving over an hour
3185  * before rollover.
3186  *
3187  * As reading the *exact* system time is too expensive to be done
3188  * whenever setting up a TCP connection, we increment the time
3189  * offset in two ways.  First, a small random positive increment
3190  * is added to isn_offset for each connection that is set up.
3191  * Second, the function tcp_isn_tick fires once per clock tick
3192  * and increments isn_offset as necessary so that sequence numbers
3193  * are incremented at approximately ISN_BYTES_PER_SECOND.  The
3194  * random positive increments serve only to ensure that the same
3195  * exact sequence number is never sent out twice (as could otherwise
3196  * happen when a port is recycled in less than the system tick
3197  * interval.)
3198  *
3199  * net.inet.tcp.isn_reseed_interval controls the number of seconds
3200  * between seeding of isn_secret.  This is normally set to zero,
3201  * as reseeding should not be necessary.
3202  *
3203  * Locking of the global variables isn_secret, isn_last_reseed, isn_offset,
3204  * isn_offset_old, and isn_ctx is performed using the ISN lock.  In
3205  * general, this means holding an exclusive (write) lock.
3206  */
3207
3208 #define ISN_BYTES_PER_SECOND 1048576
3209 #define ISN_STATIC_INCREMENT 4096
3210 #define ISN_RANDOM_INCREMENT (4096 - 1)
3211 #define ISN_SECRET_LENGTH    SIPHASH_KEY_LENGTH
3212
3213 VNET_DEFINE_STATIC(u_char, isn_secret[ISN_SECRET_LENGTH]);
3214 VNET_DEFINE_STATIC(int, isn_last);
3215 VNET_DEFINE_STATIC(int, isn_last_reseed);
3216 VNET_DEFINE_STATIC(u_int32_t, isn_offset);
3217 VNET_DEFINE_STATIC(u_int32_t, isn_offset_old);
3218
3219 #define V_isn_secret                    VNET(isn_secret)
3220 #define V_isn_last                      VNET(isn_last)
3221 #define V_isn_last_reseed               VNET(isn_last_reseed)
3222 #define V_isn_offset                    VNET(isn_offset)
3223 #define V_isn_offset_old                VNET(isn_offset_old)
3224
3225 tcp_seq
3226 tcp_new_isn(struct in_conninfo *inc)
3227 {
3228         tcp_seq new_isn;
3229         u_int32_t projected_offset;
3230
3231         ISN_LOCK();
3232         /* Seed if this is the first use, reseed if requested. */
3233         if ((V_isn_last_reseed == 0) || ((V_tcp_isn_reseed_interval > 0) &&
3234              (((u_int)V_isn_last_reseed + (u_int)V_tcp_isn_reseed_interval*hz)
3235                 < (u_int)ticks))) {
3236                 arc4rand(&V_isn_secret, sizeof(V_isn_secret), 0);
3237                 V_isn_last_reseed = ticks;
3238         }
3239
3240         /* Compute the hash and return the ISN. */
3241         new_isn = (tcp_seq)tcp_keyed_hash(inc, V_isn_secret,
3242             sizeof(V_isn_secret));
3243         V_isn_offset += ISN_STATIC_INCREMENT +
3244                 (arc4random() & ISN_RANDOM_INCREMENT);
3245         if (ticks != V_isn_last) {
3246                 projected_offset = V_isn_offset_old +
3247                     ISN_BYTES_PER_SECOND / hz * (ticks - V_isn_last);
3248                 if (SEQ_GT(projected_offset, V_isn_offset))
3249                         V_isn_offset = projected_offset;
3250                 V_isn_offset_old = V_isn_offset;
3251                 V_isn_last = ticks;
3252         }
3253         new_isn += V_isn_offset;
3254         ISN_UNLOCK();
3255         return (new_isn);
3256 }
3257
3258 /*
3259  * When a specific ICMP unreachable message is received and the
3260  * connection state is SYN-SENT, drop the connection.  This behavior
3261  * is controlled by the icmp_may_rst sysctl.
3262  */
3263 static struct inpcb *
3264 tcp_drop_syn_sent(struct inpcb *inp, int errno)
3265 {
3266         struct tcpcb *tp;
3267
3268         NET_EPOCH_ASSERT();
3269         INP_WLOCK_ASSERT(inp);
3270
3271         tp = intotcpcb(inp);
3272         if (tp->t_state != TCPS_SYN_SENT)
3273                 return (inp);
3274
3275         if (IS_FASTOPEN(tp->t_flags))
3276                 tcp_fastopen_disable_path(tp);
3277
3278         tp = tcp_drop(tp, errno);
3279         if (tp != NULL)
3280                 return (inp);
3281         else
3282                 return (NULL);
3283 }
3284
3285 /*
3286  * When `need fragmentation' ICMP is received, update our idea of the MSS
3287  * based on the new value. Also nudge TCP to send something, since we
3288  * know the packet we just sent was dropped.
3289  * This duplicates some code in the tcp_mss() function in tcp_input.c.
3290  */
3291 static struct inpcb *
3292 tcp_mtudisc_notify(struct inpcb *inp, int error)
3293 {
3294
3295         return (tcp_mtudisc(inp, -1));
3296 }
3297
3298 static struct inpcb *
3299 tcp_mtudisc(struct inpcb *inp, int mtuoffer)
3300 {
3301         struct tcpcb *tp;
3302         struct socket *so;
3303
3304         INP_WLOCK_ASSERT(inp);
3305
3306         tp = intotcpcb(inp);
3307         KASSERT(tp != NULL, ("tcp_mtudisc: tp == NULL"));
3308
3309         tcp_mss_update(tp, -1, mtuoffer, NULL, NULL);
3310
3311         so = inp->inp_socket;
3312         SOCKBUF_LOCK(&so->so_snd);
3313         /* If the mss is larger than the socket buffer, decrease the mss. */
3314         if (so->so_snd.sb_hiwat < tp->t_maxseg)
3315                 tp->t_maxseg = so->so_snd.sb_hiwat;
3316         SOCKBUF_UNLOCK(&so->so_snd);
3317
3318         TCPSTAT_INC(tcps_mturesent);
3319         tp->t_rtttime = 0;
3320         tp->snd_nxt = tp->snd_una;
3321         tcp_free_sackholes(tp);
3322         tp->snd_recover = tp->snd_max;
3323         if (tp->t_flags & TF_SACK_PERMIT)
3324                 EXIT_FASTRECOVERY(tp->t_flags);
3325         if (tp->t_fb->tfb_tcp_mtu_chg != NULL) {
3326                 /*
3327                  * Conceptually the snd_nxt setting
3328                  * and freeing sack holes should
3329                  * be done by the default stacks
3330                  * own tfb_tcp_mtu_chg().
3331                  */
3332                 tp->t_fb->tfb_tcp_mtu_chg(tp);
3333         }
3334         if (tcp_output(tp) < 0)
3335                 return (NULL);
3336         else
3337                 return (inp);
3338 }
3339
3340 #ifdef INET
3341 /*
3342  * Look-up the routing entry to the peer of this inpcb.  If no route
3343  * is found and it cannot be allocated, then return 0.  This routine
3344  * is called by TCP routines that access the rmx structure and by
3345  * tcp_mss_update to get the peer/interface MTU.
3346  */
3347 uint32_t
3348 tcp_maxmtu(struct in_conninfo *inc, struct tcp_ifcap *cap)
3349 {
3350         struct nhop_object *nh;
3351         struct ifnet *ifp;
3352         uint32_t maxmtu = 0;
3353
3354         KASSERT(inc != NULL, ("tcp_maxmtu with NULL in_conninfo pointer"));
3355
3356         if (inc->inc_faddr.s_addr != INADDR_ANY) {
3357                 nh = fib4_lookup(inc->inc_fibnum, inc->inc_faddr, 0, NHR_NONE, 0);
3358                 if (nh == NULL)
3359                         return (0);
3360
3361                 ifp = nh->nh_ifp;
3362                 maxmtu = nh->nh_mtu;
3363
3364                 /* Report additional interface capabilities. */
3365                 if (cap != NULL) {
3366                         if (ifp->if_capenable & IFCAP_TSO4 &&
3367                             ifp->if_hwassist & CSUM_TSO) {
3368                                 cap->ifcap |= CSUM_TSO;
3369                                 cap->tsomax = ifp->if_hw_tsomax;
3370                                 cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
3371                                 cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
3372                         }
3373                 }
3374         }
3375         return (maxmtu);
3376 }
3377 #endif /* INET */
3378
3379 #ifdef INET6
3380 uint32_t
3381 tcp_maxmtu6(struct in_conninfo *inc, struct tcp_ifcap *cap)
3382 {
3383         struct nhop_object *nh;
3384         struct in6_addr dst6;
3385         uint32_t scopeid;
3386         struct ifnet *ifp;
3387         uint32_t maxmtu = 0;
3388
3389         KASSERT(inc != NULL, ("tcp_maxmtu6 with NULL in_conninfo pointer"));
3390
3391         if (inc->inc_flags & INC_IPV6MINMTU)
3392                 return (IPV6_MMTU);
3393
3394         if (!IN6_IS_ADDR_UNSPECIFIED(&inc->inc6_faddr)) {
3395                 in6_splitscope(&inc->inc6_faddr, &dst6, &scopeid);
3396                 nh = fib6_lookup(inc->inc_fibnum, &dst6, scopeid, NHR_NONE, 0);
3397                 if (nh == NULL)
3398                         return (0);
3399
3400                 ifp = nh->nh_ifp;
3401                 maxmtu = nh->nh_mtu;
3402
3403                 /* Report additional interface capabilities. */
3404                 if (cap != NULL) {
3405                         if (ifp->if_capenable & IFCAP_TSO6 &&
3406                             ifp->if_hwassist & CSUM_TSO) {
3407                                 cap->ifcap |= CSUM_TSO;
3408                                 cap->tsomax = ifp->if_hw_tsomax;
3409                                 cap->tsomaxsegcount = ifp->if_hw_tsomaxsegcount;
3410                                 cap->tsomaxsegsize = ifp->if_hw_tsomaxsegsize;
3411                         }
3412                 }
3413         }
3414
3415         return (maxmtu);
3416 }
3417
3418 /*
3419  * Handle setsockopt(IPV6_USE_MIN_MTU) by a TCP stack.
3420  *
3421  * XXXGL: we are updating inpcb here with INC_IPV6MINMTU flag.
3422  * The right place to do that is ip6_setpktopt() that has just been
3423  * executed.  By the way it just filled ip6po_minmtu for us.
3424  */
3425 void
3426 tcp6_use_min_mtu(struct tcpcb *tp)
3427 {
3428         struct inpcb *inp = tptoinpcb(tp);
3429
3430         INP_WLOCK_ASSERT(inp);
3431         /*
3432          * In case of the IPV6_USE_MIN_MTU socket
3433          * option, the INC_IPV6MINMTU flag to announce
3434          * a corresponding MSS during the initial
3435          * handshake.  If the TCP connection is not in
3436          * the front states, just reduce the MSS being
3437          * used.  This avoids the sending of TCP
3438          * segments which will be fragmented at the
3439          * IPv6 layer.
3440          */
3441         inp->inp_inc.inc_flags |= INC_IPV6MINMTU;
3442         if ((tp->t_state >= TCPS_SYN_SENT) &&
3443             (inp->inp_inc.inc_flags & INC_ISIPV6)) {
3444                 struct ip6_pktopts *opt;
3445
3446                 opt = inp->in6p_outputopts;
3447                 if (opt != NULL && opt->ip6po_minmtu == IP6PO_MINMTU_ALL &&
3448                     tp->t_maxseg > TCP6_MSS)
3449                         tp->t_maxseg = TCP6_MSS;
3450         }
3451 }
3452 #endif /* INET6 */
3453
3454 /*
3455  * Calculate effective SMSS per RFC5681 definition for a given TCP
3456  * connection at its current state, taking into account SACK and etc.
3457  */
3458 u_int
3459 tcp_maxseg(const struct tcpcb *tp)
3460 {
3461         u_int optlen;
3462
3463         if (tp->t_flags & TF_NOOPT)
3464                 return (tp->t_maxseg);
3465
3466         /*
3467          * Here we have a simplified code from tcp_addoptions(),
3468          * without a proper loop, and having most of paddings hardcoded.
3469          * We might make mistakes with padding here in some edge cases,
3470          * but this is harmless, since result of tcp_maxseg() is used
3471          * only in cwnd and ssthresh estimations.
3472          */
3473         if (TCPS_HAVEESTABLISHED(tp->t_state)) {
3474                 if (tp->t_flags & TF_RCVD_TSTMP)
3475                         optlen = TCPOLEN_TSTAMP_APPA;
3476                 else
3477                         optlen = 0;
3478 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3479                 if (tp->t_flags & TF_SIGNATURE)
3480                         optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3481 #endif
3482                 if ((tp->t_flags & TF_SACK_PERMIT) && tp->rcv_numsacks > 0) {
3483                         optlen += TCPOLEN_SACKHDR;
3484                         optlen += tp->rcv_numsacks * TCPOLEN_SACK;
3485                         optlen = PADTCPOLEN(optlen);
3486                 }
3487         } else {
3488                 if (tp->t_flags & TF_REQ_TSTMP)
3489                         optlen = TCPOLEN_TSTAMP_APPA;
3490                 else
3491                         optlen = PADTCPOLEN(TCPOLEN_MAXSEG);
3492                 if (tp->t_flags & TF_REQ_SCALE)
3493                         optlen += PADTCPOLEN(TCPOLEN_WINDOW);
3494 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3495                 if (tp->t_flags & TF_SIGNATURE)
3496                         optlen += PADTCPOLEN(TCPOLEN_SIGNATURE);
3497 #endif
3498                 if (tp->t_flags & TF_SACK_PERMIT)
3499                         optlen += PADTCPOLEN(TCPOLEN_SACK_PERMITTED);
3500         }
3501 #undef PAD
3502         optlen = min(optlen, TCP_MAXOLEN);
3503         return (tp->t_maxseg - optlen);
3504 }
3505
3506
3507 u_int
3508 tcp_fixed_maxseg(const struct tcpcb *tp)
3509 {
3510         int optlen;
3511
3512         if (tp->t_flags & TF_NOOPT)
3513                 return (tp->t_maxseg);
3514
3515         /*
3516          * Here we have a simplified code from tcp_addoptions(),
3517          * without a proper loop, and having most of paddings hardcoded.
3518          * We only consider fixed options that we would send every
3519          * time I.e. SACK is not considered. This is important
3520          * for cc modules to figure out what the modulo of the
3521          * cwnd should be.
3522          */
3523 #define PAD(len)        ((((len) / 4) + !!((len) % 4)) * 4)
3524         if (TCPS_HAVEESTABLISHED(tp->t_state)) {
3525                 if (tp->t_flags & TF_RCVD_TSTMP)
3526                         optlen = TCPOLEN_TSTAMP_APPA;
3527                 else
3528                         optlen = 0;
3529 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3530                 if (tp->t_flags & TF_SIGNATURE)
3531                         optlen += PAD(TCPOLEN_SIGNATURE);
3532 #endif
3533         } else {
3534                 if (tp->t_flags & TF_REQ_TSTMP)
3535                         optlen = TCPOLEN_TSTAMP_APPA;
3536                 else
3537                         optlen = PAD(TCPOLEN_MAXSEG);
3538                 if (tp->t_flags & TF_REQ_SCALE)
3539                         optlen += PAD(TCPOLEN_WINDOW);
3540 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
3541                 if (tp->t_flags & TF_SIGNATURE)
3542                         optlen += PAD(TCPOLEN_SIGNATURE);
3543 #endif
3544                 if (tp->t_flags & TF_SACK_PERMIT)
3545                         optlen += PAD(TCPOLEN_SACK_PERMITTED);
3546         }
3547 #undef PAD
3548         optlen = min(optlen, TCP_MAXOLEN);
3549         return (tp->t_maxseg - optlen);
3550 }
3551
3552
3553
3554 static int
3555 sysctl_drop(SYSCTL_HANDLER_ARGS)
3556 {
3557         /* addrs[0] is a foreign socket, addrs[1] is a local one. */
3558         struct sockaddr_storage addrs[2];
3559         struct inpcb *inp;
3560         struct tcpcb *tp;
3561 #ifdef INET
3562         struct sockaddr_in *fin = NULL, *lin = NULL;
3563 #endif
3564         struct epoch_tracker et;
3565 #ifdef INET6
3566         struct sockaddr_in6 *fin6, *lin6;
3567 #endif
3568         int error;
3569
3570         inp = NULL;
3571 #ifdef INET6
3572         fin6 = lin6 = NULL;
3573 #endif
3574         error = 0;
3575
3576         if (req->oldptr != NULL || req->oldlen != 0)
3577                 return (EINVAL);
3578         if (req->newptr == NULL)
3579                 return (EPERM);
3580         if (req->newlen < sizeof(addrs))
3581                 return (ENOMEM);
3582         error = SYSCTL_IN(req, &addrs, sizeof(addrs));
3583         if (error)
3584                 return (error);
3585
3586         switch (addrs[0].ss_family) {
3587 #ifdef INET6
3588         case AF_INET6:
3589                 fin6 = (struct sockaddr_in6 *)&addrs[0];
3590                 lin6 = (struct sockaddr_in6 *)&addrs[1];
3591                 if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
3592                     lin6->sin6_len != sizeof(struct sockaddr_in6))
3593                         return (EINVAL);
3594                 if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
3595                         if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
3596                                 return (EINVAL);
3597                         in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
3598                         in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
3599 #ifdef INET
3600                         fin = (struct sockaddr_in *)&addrs[0];
3601                         lin = (struct sockaddr_in *)&addrs[1];
3602 #endif
3603                         break;
3604                 }
3605                 error = sa6_embedscope(fin6, V_ip6_use_defzone);
3606                 if (error)
3607                         return (error);
3608                 error = sa6_embedscope(lin6, V_ip6_use_defzone);
3609                 if (error)
3610                         return (error);
3611                 break;
3612 #endif
3613 #ifdef INET
3614         case AF_INET:
3615                 fin = (struct sockaddr_in *)&addrs[0];
3616                 lin = (struct sockaddr_in *)&addrs[1];
3617                 if (fin->sin_len != sizeof(struct sockaddr_in) ||
3618                     lin->sin_len != sizeof(struct sockaddr_in))
3619                         return (EINVAL);
3620                 break;
3621 #endif
3622         default:
3623                 return (EINVAL);
3624         }
3625         NET_EPOCH_ENTER(et);
3626         switch (addrs[0].ss_family) {
3627 #ifdef INET6
3628         case AF_INET6:
3629                 inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
3630                     fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
3631                     INPLOOKUP_WLOCKPCB, NULL);
3632                 break;
3633 #endif
3634 #ifdef INET
3635         case AF_INET:
3636                 inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
3637                     lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
3638                 break;
3639 #endif
3640         }
3641         if (inp != NULL) {
3642                 if (!SOLISTENING(inp->inp_socket)) {
3643                         tp = intotcpcb(inp);
3644                         tp = tcp_drop(tp, ECONNABORTED);
3645                         if (tp != NULL)
3646                                 INP_WUNLOCK(inp);
3647                 } else
3648                         INP_WUNLOCK(inp);
3649         } else
3650                 error = ESRCH;
3651         NET_EPOCH_EXIT(et);
3652         return (error);
3653 }
3654
3655 SYSCTL_PROC(_net_inet_tcp, TCPCTL_DROP, drop,
3656     CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3657     CTLFLAG_NEEDGIANT, NULL, 0, sysctl_drop, "",
3658     "Drop TCP connection");
3659
3660 static int
3661 tcp_sysctl_setsockopt(SYSCTL_HANDLER_ARGS)
3662 {
3663         return (sysctl_setsockopt(oidp, arg1, arg2, req, &V_tcbinfo,
3664             &tcp_ctloutput_set));
3665 }
3666
3667 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, setsockopt,
3668     CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3669     CTLFLAG_MPSAFE, NULL, 0, tcp_sysctl_setsockopt, "",
3670     "Set socket option for TCP endpoint");
3671
3672 #ifdef KERN_TLS
3673 static int
3674 sysctl_switch_tls(SYSCTL_HANDLER_ARGS)
3675 {
3676         /* addrs[0] is a foreign socket, addrs[1] is a local one. */
3677         struct sockaddr_storage addrs[2];
3678         struct inpcb *inp;
3679 #ifdef INET
3680         struct sockaddr_in *fin = NULL, *lin = NULL;
3681 #endif
3682         struct epoch_tracker et;
3683 #ifdef INET6
3684         struct sockaddr_in6 *fin6, *lin6;
3685 #endif
3686         int error;
3687
3688         inp = NULL;
3689 #ifdef INET6
3690         fin6 = lin6 = NULL;
3691 #endif
3692         error = 0;
3693
3694         if (req->oldptr != NULL || req->oldlen != 0)
3695                 return (EINVAL);
3696         if (req->newptr == NULL)
3697                 return (EPERM);
3698         if (req->newlen < sizeof(addrs))
3699                 return (ENOMEM);
3700         error = SYSCTL_IN(req, &addrs, sizeof(addrs));
3701         if (error)
3702                 return (error);
3703
3704         switch (addrs[0].ss_family) {
3705 #ifdef INET6
3706         case AF_INET6:
3707                 fin6 = (struct sockaddr_in6 *)&addrs[0];
3708                 lin6 = (struct sockaddr_in6 *)&addrs[1];
3709                 if (fin6->sin6_len != sizeof(struct sockaddr_in6) ||
3710                     lin6->sin6_len != sizeof(struct sockaddr_in6))
3711                         return (EINVAL);
3712                 if (IN6_IS_ADDR_V4MAPPED(&fin6->sin6_addr)) {
3713                         if (!IN6_IS_ADDR_V4MAPPED(&lin6->sin6_addr))
3714                                 return (EINVAL);
3715                         in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[0]);
3716                         in6_sin6_2_sin_in_sock((struct sockaddr *)&addrs[1]);
3717 #ifdef INET
3718                         fin = (struct sockaddr_in *)&addrs[0];
3719                         lin = (struct sockaddr_in *)&addrs[1];
3720 #endif
3721                         break;
3722                 }
3723                 error = sa6_embedscope(fin6, V_ip6_use_defzone);
3724                 if (error)
3725                         return (error);
3726                 error = sa6_embedscope(lin6, V_ip6_use_defzone);
3727                 if (error)
3728                         return (error);
3729                 break;
3730 #endif
3731 #ifdef INET
3732         case AF_INET:
3733                 fin = (struct sockaddr_in *)&addrs[0];
3734                 lin = (struct sockaddr_in *)&addrs[1];
3735                 if (fin->sin_len != sizeof(struct sockaddr_in) ||
3736                     lin->sin_len != sizeof(struct sockaddr_in))
3737                         return (EINVAL);
3738                 break;
3739 #endif
3740         default:
3741                 return (EINVAL);
3742         }
3743         NET_EPOCH_ENTER(et);
3744         switch (addrs[0].ss_family) {
3745 #ifdef INET6
3746         case AF_INET6:
3747                 inp = in6_pcblookup(&V_tcbinfo, &fin6->sin6_addr,
3748                     fin6->sin6_port, &lin6->sin6_addr, lin6->sin6_port,
3749                     INPLOOKUP_WLOCKPCB, NULL);
3750                 break;
3751 #endif
3752 #ifdef INET
3753         case AF_INET:
3754                 inp = in_pcblookup(&V_tcbinfo, fin->sin_addr, fin->sin_port,
3755                     lin->sin_addr, lin->sin_port, INPLOOKUP_WLOCKPCB, NULL);
3756                 break;
3757 #endif
3758         }
3759         NET_EPOCH_EXIT(et);
3760         if (inp != NULL) {
3761                 struct socket *so;
3762
3763                 so = inp->inp_socket;
3764                 soref(so);
3765                 error = ktls_set_tx_mode(so,
3766                     arg2 == 0 ? TCP_TLS_MODE_SW : TCP_TLS_MODE_IFNET);
3767                 INP_WUNLOCK(inp);
3768                 sorele(so);
3769         } else
3770                 error = ESRCH;
3771         return (error);
3772 }
3773
3774 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_sw_tls,
3775     CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3776     CTLFLAG_NEEDGIANT, NULL, 0, sysctl_switch_tls, "",
3777     "Switch TCP connection to SW TLS");
3778 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, switch_to_ifnet_tls,
3779     CTLFLAG_VNET | CTLTYPE_STRUCT | CTLFLAG_WR | CTLFLAG_SKIP |
3780     CTLFLAG_NEEDGIANT, NULL, 1, sysctl_switch_tls, "",
3781     "Switch TCP connection to ifnet TLS");
3782 #endif
3783
3784 /*
3785  * Generate a standardized TCP log line for use throughout the
3786  * tcp subsystem.  Memory allocation is done with M_NOWAIT to
3787  * allow use in the interrupt context.
3788  *
3789  * NB: The caller MUST free(s, M_TCPLOG) the returned string.
3790  * NB: The function may return NULL if memory allocation failed.
3791  *
3792  * Due to header inclusion and ordering limitations the struct ip
3793  * and ip6_hdr pointers have to be passed as void pointers.
3794  */
3795 char *
3796 tcp_log_vain(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3797     const void *ip6hdr)
3798 {
3799
3800         /* Is logging enabled? */
3801         if (V_tcp_log_in_vain == 0)
3802                 return (NULL);
3803
3804         return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
3805 }
3806
3807 char *
3808 tcp_log_addrs(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3809     const void *ip6hdr)
3810 {
3811
3812         /* Is logging enabled? */
3813         if (tcp_log_debug == 0)
3814                 return (NULL);
3815
3816         return (tcp_log_addr(inc, th, ip4hdr, ip6hdr));
3817 }
3818
3819 static char *
3820 tcp_log_addr(struct in_conninfo *inc, struct tcphdr *th, const void *ip4hdr,
3821     const void *ip6hdr)
3822 {
3823         char *s, *sp;
3824         size_t size;
3825 #ifdef INET
3826         const struct ip *ip = (const struct ip *)ip4hdr;
3827 #endif
3828 #ifdef INET6
3829         const struct ip6_hdr *ip6 = (const struct ip6_hdr *)ip6hdr;
3830 #endif /* INET6 */
3831
3832         /*
3833          * The log line looks like this:
3834          * "TCP: [1.2.3.4]:50332 to [1.2.3.4]:80 tcpflags 0x2<SYN>"
3835          */
3836         size = sizeof("TCP: []:12345 to []:12345 tcpflags 0x2<>") +
3837             sizeof(PRINT_TH_FLAGS) + 1 +
3838 #ifdef INET6
3839             2 * INET6_ADDRSTRLEN;
3840 #else
3841             2 * INET_ADDRSTRLEN;
3842 #endif /* INET6 */
3843
3844         s = malloc(size, M_TCPLOG, M_ZERO|M_NOWAIT);
3845         if (s == NULL)
3846                 return (NULL);
3847
3848         strcat(s, "TCP: [");
3849         sp = s + strlen(s);
3850
3851         if (inc && ((inc->inc_flags & INC_ISIPV6) == 0)) {
3852                 inet_ntoa_r(inc->inc_faddr, sp);
3853                 sp = s + strlen(s);
3854                 sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
3855                 sp = s + strlen(s);
3856                 inet_ntoa_r(inc->inc_laddr, sp);
3857                 sp = s + strlen(s);
3858                 sprintf(sp, "]:%i", ntohs(inc->inc_lport));
3859 #ifdef INET6
3860         } else if (inc) {
3861                 ip6_sprintf(sp, &inc->inc6_faddr);
3862                 sp = s + strlen(s);
3863                 sprintf(sp, "]:%i to [", ntohs(inc->inc_fport));
3864                 sp = s + strlen(s);
3865                 ip6_sprintf(sp, &inc->inc6_laddr);
3866                 sp = s + strlen(s);
3867                 sprintf(sp, "]:%i", ntohs(inc->inc_lport));
3868         } else if (ip6 && th) {
3869                 ip6_sprintf(sp, &ip6->ip6_src);
3870                 sp = s + strlen(s);
3871                 sprintf(sp, "]:%i to [", ntohs(th->th_sport));
3872                 sp = s + strlen(s);
3873                 ip6_sprintf(sp, &ip6->ip6_dst);
3874                 sp = s + strlen(s);
3875                 sprintf(sp, "]:%i", ntohs(th->th_dport));
3876 #endif /* INET6 */
3877 #ifdef INET
3878         } else if (ip && th) {
3879                 inet_ntoa_r(ip->ip_src, sp);
3880                 sp = s + strlen(s);
3881                 sprintf(sp, "]:%i to [", ntohs(th->th_sport));
3882                 sp = s + strlen(s);
3883                 inet_ntoa_r(ip->ip_dst, sp);
3884                 sp = s + strlen(s);
3885                 sprintf(sp, "]:%i", ntohs(th->th_dport));
3886 #endif /* INET */
3887         } else {
3888                 free(s, M_TCPLOG);
3889                 return (NULL);
3890         }
3891         sp = s + strlen(s);
3892         if (th)
3893                 sprintf(sp, " tcpflags 0x%b", tcp_get_flags(th), PRINT_TH_FLAGS);
3894         if (*(s + size - 1) != '\0')
3895                 panic("%s: string too long", __func__);
3896         return (s);
3897 }
3898
3899 /*
3900  * A subroutine which makes it easy to track TCP state changes with DTrace.
3901  * This function shouldn't be called for t_state initializations that don't
3902  * correspond to actual TCP state transitions.
3903  */
3904 void
3905 tcp_state_change(struct tcpcb *tp, int newstate)
3906 {
3907 #if defined(KDTRACE_HOOKS)
3908         int pstate = tp->t_state;
3909 #endif
3910
3911         TCPSTATES_DEC(tp->t_state);
3912         TCPSTATES_INC(newstate);
3913         tp->t_state = newstate;
3914         TCP_PROBE6(state__change, NULL, tp, NULL, tp, NULL, pstate);
3915 }
3916
3917 /*
3918  * Create an external-format (``xtcpcb'') structure using the information in
3919  * the kernel-format tcpcb structure pointed to by tp.  This is done to
3920  * reduce the spew of irrelevant information over this interface, to isolate
3921  * user code from changes in the kernel structure, and potentially to provide
3922  * information-hiding if we decide that some of this information should be
3923  * hidden from users.
3924  */
3925 void
3926 tcp_inptoxtp(const struct inpcb *inp, struct xtcpcb *xt)
3927 {
3928         struct tcpcb *tp = intotcpcb(inp);
3929         sbintime_t now;
3930
3931         bzero(xt, sizeof(*xt));
3932         xt->t_state = tp->t_state;
3933         xt->t_logstate = tcp_get_bblog_state(tp);
3934         xt->t_flags = tp->t_flags;
3935         xt->t_sndzerowin = tp->t_sndzerowin;
3936         xt->t_sndrexmitpack = tp->t_sndrexmitpack;
3937         xt->t_rcvoopack = tp->t_rcvoopack;
3938         xt->t_rcv_wnd = tp->rcv_wnd;
3939         xt->t_snd_wnd = tp->snd_wnd;
3940         xt->t_snd_cwnd = tp->snd_cwnd;
3941         xt->t_snd_ssthresh = tp->snd_ssthresh;
3942         xt->t_dsack_bytes = tp->t_dsack_bytes;
3943         xt->t_dsack_tlp_bytes = tp->t_dsack_tlp_bytes;
3944         xt->t_dsack_pack = tp->t_dsack_pack;
3945         xt->t_maxseg = tp->t_maxseg;
3946         xt->xt_ecn = (tp->t_flags2 & TF2_ECN_PERMIT) ? 1 : 0 +
3947                      (tp->t_flags2 & TF2_ACE_PERMIT) ? 2 : 0;
3948
3949         now = getsbinuptime();
3950 #define COPYTIMER(which,where)  do {                                    \
3951         if (tp->t_timers[which] != SBT_MAX)                             \
3952                 xt->where = (tp->t_timers[which] - now) / SBT_1MS;      \
3953         else                                                            \
3954                 xt->where = 0;                                          \
3955 } while (0)
3956         COPYTIMER(TT_DELACK, tt_delack);
3957         COPYTIMER(TT_REXMT, tt_rexmt);
3958         COPYTIMER(TT_PERSIST, tt_persist);
3959         COPYTIMER(TT_KEEP, tt_keep);
3960         COPYTIMER(TT_2MSL, tt_2msl);
3961 #undef COPYTIMER
3962         xt->t_rcvtime = 1000 * (ticks - tp->t_rcvtime) / hz;
3963
3964         xt->xt_encaps_port = tp->t_port;
3965         bcopy(tp->t_fb->tfb_tcp_block_name, xt->xt_stack,
3966             TCP_FUNCTION_NAME_LEN_MAX);
3967         bcopy(CC_ALGO(tp)->name, xt->xt_cc, TCP_CA_NAME_MAX);
3968 #ifdef TCP_BLACKBOX
3969         (void)tcp_log_get_id(tp, xt->xt_logid);
3970 #endif
3971
3972         xt->xt_len = sizeof(struct xtcpcb);
3973         in_pcbtoxinpcb(inp, &xt->xt_inp);
3974         /*
3975          * TCP doesn't use inp_ppcb pointer, we embed inpcb into tcpcb.
3976          * Fixup the pointer that in_pcbtoxinpcb() has set.  When printing
3977          * TCP netstat(1) used to use this pointer, so this fixup needs to
3978          * stay for stable/14.
3979          */
3980         xt->xt_inp.inp_ppcb = (uintptr_t)tp;
3981 }
3982
3983 void
3984 tcp_log_end_status(struct tcpcb *tp, uint8_t status)
3985 {
3986         uint32_t bit, i;
3987
3988         if ((tp == NULL) ||
3989             (status > TCP_EI_STATUS_MAX_VALUE) ||
3990             (status == 0)) {
3991                 /* Invalid */
3992                 return;
3993         }
3994         if (status > (sizeof(uint32_t) * 8)) {
3995                 /* Should this be a KASSERT? */
3996                 return;
3997         }
3998         bit = 1U << (status - 1);
3999         if (bit & tp->t_end_info_status) {
4000                 /* already logged */
4001                 return;
4002         }
4003         for (i = 0; i < TCP_END_BYTE_INFO; i++) {
4004                 if (tp->t_end_info_bytes[i] == TCP_EI_EMPTY_SLOT) {
4005                         tp->t_end_info_bytes[i] = status;
4006                         tp->t_end_info_status |= bit;
4007                         break;
4008                 }
4009         }
4010 }
4011
4012 int
4013 tcp_can_enable_pacing(void)
4014 {
4015
4016         if ((tcp_pacing_limit == -1) ||
4017             (tcp_pacing_limit > number_of_tcp_connections_pacing)) {
4018                 atomic_fetchadd_int(&number_of_tcp_connections_pacing, 1);
4019                 shadow_num_connections = number_of_tcp_connections_pacing;
4020                 return (1);
4021         } else {
4022                 counter_u64_add(tcp_pacing_failures, 1);
4023                 return (0);
4024         }
4025 }
4026
4027 static uint8_t tcp_pacing_warning = 0;
4028
4029 void
4030 tcp_decrement_paced_conn(void)
4031 {
4032         uint32_t ret;
4033
4034         ret = atomic_fetchadd_int(&number_of_tcp_connections_pacing, -1);
4035         shadow_num_connections = number_of_tcp_connections_pacing;
4036         KASSERT(ret != 0, ("tcp_paced_connection_exits -1 would cause wrap?"));
4037         if (ret == 0) {
4038                 if (tcp_pacing_limit != -1) {
4039                         printf("Warning all pacing is now disabled, count decrements invalidly!\n");
4040                         tcp_pacing_limit = 0;
4041                 } else if (tcp_pacing_warning == 0) {
4042                         printf("Warning pacing count is invalid, invalid decrement\n");
4043                         tcp_pacing_warning = 1;
4044                 }
4045         }
4046 }
4047
4048 static void
4049 tcp_default_switch_failed(struct tcpcb *tp)
4050 {
4051         /*
4052          * If a switch fails we only need to
4053          * care about two things:
4054          * a) The t_flags2
4055          * and
4056          * b) The timer granularity.
4057          * Timeouts, at least for now, don't use the
4058          * old callout system in the other stacks so
4059          * those are hopefully safe.
4060          */
4061         tcp_lro_features_off(tp);
4062         tcp_change_time_units(tp, TCP_TMR_GRANULARITY_TICKS);
4063 }
4064
4065 #ifdef TCP_ACCOUNTING
4066 int
4067 tcp_do_ack_accounting(struct tcpcb *tp, struct tcphdr *th, struct tcpopt *to, uint32_t tiwin, int mss)
4068 {
4069         if (SEQ_LT(th->th_ack, tp->snd_una)) {
4070                 /* Do we have a SACK? */
4071                 if (to->to_flags & TOF_SACK) {
4072                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4073                                 tp->tcp_cnt_counters[ACK_SACK]++;
4074                         }
4075                         return (ACK_SACK);
4076                 } else {
4077                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4078                                 tp->tcp_cnt_counters[ACK_BEHIND]++;
4079                         }
4080                         return (ACK_BEHIND);
4081                 }
4082         } else if (th->th_ack == tp->snd_una) {
4083                 /* Do we have a SACK? */
4084                 if (to->to_flags & TOF_SACK) {
4085                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4086                                 tp->tcp_cnt_counters[ACK_SACK]++;
4087                         }
4088                         return (ACK_SACK);
4089                 } else if (tiwin != tp->snd_wnd) {
4090                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4091                                 tp->tcp_cnt_counters[ACK_RWND]++;
4092                         }
4093                         return (ACK_RWND);
4094                 } else {
4095                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4096                                 tp->tcp_cnt_counters[ACK_DUPACK]++;
4097                         }
4098                         return (ACK_DUPACK);
4099                 }
4100         } else {
4101                 if (!SEQ_GT(th->th_ack, tp->snd_max)) {
4102                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4103                                 tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((th->th_ack - tp->snd_una) + mss - 1)/mss);
4104                         }
4105                 }
4106                 if (to->to_flags & TOF_SACK) {
4107                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4108                                 tp->tcp_cnt_counters[ACK_CUMACK_SACK]++;
4109                         }
4110                         return (ACK_CUMACK_SACK);
4111                 } else {
4112                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
4113                                 tp->tcp_cnt_counters[ACK_CUMACK]++;
4114                         }
4115                         return (ACK_CUMACK);
4116                 }
4117         }
4118 }
4119 #endif
4120
4121 void
4122 tcp_change_time_units(struct tcpcb *tp, int granularity)
4123 {
4124         if (tp->t_tmr_granularity == granularity) {
4125                 /* We are there */
4126                 return;
4127         }
4128         if (granularity == TCP_TMR_GRANULARITY_USEC) {
4129                 KASSERT((tp->t_tmr_granularity == TCP_TMR_GRANULARITY_TICKS),
4130                         ("Granularity is not TICKS its %u in tp:%p",
4131                          tp->t_tmr_granularity, tp));
4132                 tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow);
4133                 if (tp->t_srtt > 1) {
4134                         uint32_t val, frac;
4135
4136                         val = tp->t_srtt >> TCP_RTT_SHIFT;
4137                         frac = tp->t_srtt & 0x1f;
4138                         tp->t_srtt = TICKS_2_USEC(val);
4139                         /*
4140                          * frac is the fractional part of the srtt (if any)
4141                          * but its in ticks and every bit represents
4142                          * 1/32nd of a hz.
4143                          */
4144                         if (frac) {
4145                                 if (hz == 1000) {
4146                                         frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
4147                                 } else {
4148                                         frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
4149                                 }
4150                                 tp->t_srtt += frac;
4151                         }
4152                 }
4153                 if (tp->t_rttvar) {
4154                         uint32_t val, frac;
4155
4156                         val = tp->t_rttvar >> TCP_RTTVAR_SHIFT;
4157                         frac = tp->t_rttvar & 0x1f;
4158                         tp->t_rttvar = TICKS_2_USEC(val);
4159                         /*
4160                          * frac is the fractional part of the srtt (if any)
4161                          * but its in ticks and every bit represents
4162                          * 1/32nd of a hz.
4163                          */
4164                         if (frac) {
4165                                 if (hz == 1000) {
4166                                         frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
4167                                 } else {
4168                                         frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
4169                                 }
4170                                 tp->t_rttvar += frac;
4171                         }
4172                 }
4173                 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_USEC;
4174         } else if (granularity == TCP_TMR_GRANULARITY_TICKS) {
4175                 /* Convert back to ticks, with  */
4176                 KASSERT((tp->t_tmr_granularity == TCP_TMR_GRANULARITY_USEC),
4177                         ("Granularity is not USEC its %u in tp:%p",
4178                          tp->t_tmr_granularity, tp));
4179                 if (tp->t_srtt > 1) {
4180                         uint32_t val, frac;
4181
4182                         val = USEC_2_TICKS(tp->t_srtt);
4183                         frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz);
4184                         tp->t_srtt = val << TCP_RTT_SHIFT;
4185                         /*
4186                          * frac is the fractional part here is left
4187                          * over from converting to hz and shifting.
4188                          * We need to convert this to the 5 bit
4189                          * remainder.
4190                          */
4191                         if (frac) {
4192                                 if (hz == 1000) {
4193                                         frac = (((uint64_t)frac *  (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
4194                                 } else {
4195                                         frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
4196                                 }
4197                                 tp->t_srtt += frac;
4198                         }
4199                 }
4200                 if (tp->t_rttvar) {
4201                         uint32_t val, frac;
4202
4203                         val = USEC_2_TICKS(tp->t_rttvar);
4204                         frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz);
4205                         tp->t_rttvar = val <<  TCP_RTTVAR_SHIFT;
4206                         /*
4207                          * frac is the fractional part here is left
4208                          * over from converting to hz and shifting.
4209                          * We need to convert this to the 5 bit
4210                          * remainder.
4211                          */
4212                         if (frac) {
4213                                 if (hz == 1000) {
4214                                         frac = (((uint64_t)frac *  (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
4215                                 } else {
4216                                         frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
4217                                 }
4218                                 tp->t_rttvar += frac;
4219                         }
4220                 }
4221                 tp->t_rttlow = USEC_2_TICKS(tp->t_rttlow);
4222                 tp->t_tmr_granularity = TCP_TMR_GRANULARITY_TICKS;
4223         }
4224 #ifdef INVARIANTS
4225         else {
4226                 panic("Unknown granularity:%d tp:%p",
4227                       granularity, tp);
4228         }
4229 #endif
4230 }
4231
4232 void
4233 tcp_handle_orphaned_packets(struct tcpcb *tp)
4234 {
4235         struct mbuf *save, *m, *prev;
4236         /*
4237          * Called when a stack switch is occuring from the fini()
4238          * of the old stack. We assue the init() as already been
4239          * run of the new stack and it has set the t_flags2 to
4240          * what it supports. This function will then deal with any
4241          * differences i.e. cleanup packets that maybe queued that
4242          * the newstack does not support.
4243          */
4244
4245         if (tp->t_flags2 & TF2_MBUF_L_ACKS)
4246                 return;
4247         if ((tp->t_flags2 & TF2_SUPPORTS_MBUFQ) == 0 &&
4248             !STAILQ_EMPTY(&tp->t_inqueue)) {
4249                 /*
4250                  * It is unsafe to process the packets since a
4251                  * reset may be lurking in them (its rare but it
4252                  * can occur). If we were to find a RST, then we
4253                  * would end up dropping the connection and the
4254                  * INP lock, so when we return the caller (tcp_usrreq)
4255                  * will blow up when it trys to unlock the inp.
4256                  * This new stack does not do any fancy LRO features
4257                  * so all we can do is toss the packets.
4258                  */
4259                 m = STAILQ_FIRST(&tp->t_inqueue);
4260                 STAILQ_INIT(&tp->t_inqueue);
4261                 STAILQ_FOREACH_FROM_SAFE(m, &tp->t_inqueue, m_stailqpkt, save)
4262                         m_freem(m);
4263         } else {
4264                 /*
4265                  * Here we have a stack that does mbuf queuing but
4266                  * does not support compressed ack's. We must
4267                  * walk all the mbufs and discard any compressed acks.
4268                  */
4269                 STAILQ_FOREACH_SAFE(m, &tp->t_inqueue, m_stailqpkt, save) {
4270                         if (m->m_flags & M_ACKCMP) {
4271                                 if (m == STAILQ_FIRST(&tp->t_inqueue))
4272                                         STAILQ_REMOVE_HEAD(&tp->t_inqueue,
4273                                             m_stailqpkt);
4274                                 else
4275                                         STAILQ_REMOVE_AFTER(&tp->t_inqueue,
4276                                             prev, m_stailqpkt);
4277                                 m_freem(m);
4278                         } else
4279                                 prev = m;
4280                 }
4281         }
4282 }
4283
4284 #ifdef TCP_REQUEST_TRK
4285 uint32_t
4286 tcp_estimate_tls_overhead(struct socket *so, uint64_t tls_usr_bytes)
4287 {
4288 #ifdef KERN_TLS
4289         struct ktls_session *tls;
4290         uint32_t rec_oh, records;
4291
4292         tls = so->so_snd.sb_tls_info;
4293         if (tls == NULL)
4294             return (0);
4295
4296         rec_oh = tls->params.tls_hlen + tls->params.tls_tlen;
4297         records = ((tls_usr_bytes + tls->params.max_frame_len - 1)/tls->params.max_frame_len);
4298         return (records * rec_oh);
4299 #else
4300         return (0);
4301 #endif
4302 }
4303
4304 extern uint32_t tcp_stale_entry_time;
4305 uint32_t tcp_stale_entry_time = 250000;
4306 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, usrlog_stale, CTLFLAG_RW,
4307     &tcp_stale_entry_time, 250000, "Time that a http entry without a sendfile ages out");
4308
4309 void
4310 tcp_http_log_req_info(struct tcpcb *tp, struct http_sendfile_track *http,
4311     uint16_t slot, uint8_t val, uint64_t offset, uint64_t nbytes)
4312 {
4313         if (tcp_bblogging_on(tp)) {
4314                 union tcp_log_stackspecific log;
4315                 struct timeval tv;
4316
4317                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
4318 #ifdef TCPHPTS
4319                 log.u_bbr.inhpts = tcp_in_hpts(tptoinpcb(tp));
4320 #endif
4321                 log.u_bbr.flex8 = val;
4322                 log.u_bbr.rttProp = http->timestamp;
4323                 log.u_bbr.delRate = http->start;
4324                 log.u_bbr.cur_del_rate = http->end;
4325                 log.u_bbr.flex1 = http->start_seq;
4326                 log.u_bbr.flex2 = http->end_seq;
4327                 log.u_bbr.flex3 = http->flags;
4328                 log.u_bbr.flex4 = ((http->localtime >> 32) & 0x00000000ffffffff);
4329                 log.u_bbr.flex5 = (http->localtime & 0x00000000ffffffff);
4330                 log.u_bbr.flex7 = slot;
4331                 log.u_bbr.bw_inuse = offset;
4332                 /* nbytes = flex6 | epoch */
4333                 log.u_bbr.flex6 = ((nbytes >> 32) & 0x00000000ffffffff);
4334                 log.u_bbr.epoch = (nbytes & 0x00000000ffffffff);
4335                 /* cspr =  lt_epoch | pkts_out */
4336                 log.u_bbr.lt_epoch = ((http->cspr >> 32) & 0x00000000ffffffff);
4337                 log.u_bbr.pkts_out |= (http->cspr & 0x00000000ffffffff);
4338                 log.u_bbr.applimited = tp->t_http_closed;
4339                 log.u_bbr.applimited <<= 8;
4340                 log.u_bbr.applimited |= tp->t_http_open;
4341                 log.u_bbr.applimited <<= 8;
4342                 log.u_bbr.applimited |= tp->t_http_req;
4343                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
4344                 TCP_LOG_EVENTP(tp, NULL,
4345                     &tptosocket(tp)->so_rcv,
4346                     &tptosocket(tp)->so_snd,
4347                     TCP_LOG_HTTP_T, 0,
4348                     0, &log, false, &tv);
4349         }
4350 }
4351
4352 void
4353 tcp_http_free_a_slot(struct tcpcb *tp, struct http_sendfile_track *ent)
4354 {
4355         if (tp->t_http_req > 0)
4356                 tp->t_http_req--;
4357         if (ent->flags & TCP_HTTP_TRACK_FLG_OPEN) {
4358                 if (tp->t_http_open > 0)
4359                         tp->t_http_open--;
4360         } else {
4361                 if (tp->t_http_closed > 0)
4362                         tp->t_http_closed--;
4363         }
4364         ent->flags = TCP_HTTP_TRACK_FLG_EMPTY;
4365 }
4366
4367 static void
4368 tcp_http_check_for_stale_entries(struct tcpcb *tp, uint64_t ts, int rm_oldest)
4369 {
4370         struct http_sendfile_track *ent;
4371         uint64_t time_delta, oldest_delta;
4372         int i, oldest, oldest_set = 0, cnt_rm = 0;
4373
4374         for(i = 0; i < MAX_TCP_HTTP_REQ; i++) {
4375                 ent = &tp->t_http_info[i];
4376                 if (ent->flags != TCP_HTTP_TRACK_FLG_USED) {
4377                         /*
4378                          * We only care about closed end ranges
4379                          * that are allocated and have no sendfile
4380                          * ever touching them. They would be in
4381                          * state USED.
4382                          */
4383                         continue;
4384                 }
4385                 if (ts >= ent->localtime)
4386                         time_delta = ts - ent->localtime;
4387                 else
4388                         time_delta = 0;
4389                 if (time_delta &&
4390                     ((oldest_delta < time_delta) || (oldest_set == 0))) {
4391                         oldest_set = 1;
4392                         oldest = i;
4393                         oldest_delta = time_delta;
4394                 }
4395                 if (tcp_stale_entry_time && (time_delta >= tcp_stale_entry_time)) {
4396                         /*
4397                          * No sendfile in a our time-limit
4398                          * time to purge it.
4399                          */
4400                         cnt_rm++;
4401                         tcp_http_log_req_info(tp, &tp->t_http_info[i], i, TCP_HTTP_REQ_LOG_STALE,
4402                                               time_delta, 0);
4403                         tcp_http_free_a_slot(tp, ent);
4404                 }
4405         }
4406         if ((cnt_rm == 0) && rm_oldest && oldest_set) {
4407                 ent = &tp->t_http_info[oldest];
4408                 tcp_http_log_req_info(tp, &tp->t_http_info[i], i, TCP_HTTP_REQ_LOG_STALE,
4409                                       oldest_delta, 1);
4410                 tcp_http_free_a_slot(tp, ent);
4411         }
4412 }
4413
4414 int
4415 tcp_http_check_for_comp(struct tcpcb *tp, tcp_seq ack_point)
4416 {
4417         int i, ret=0;
4418         struct http_sendfile_track *ent;
4419
4420         /* Clean up any old closed end requests that are now completed */
4421         if (tp->t_http_req == 0)
4422                 return(0);
4423         if (tp->t_http_closed == 0)
4424                 return(0);
4425         for(i = 0; i < MAX_TCP_HTTP_REQ; i++) {
4426                 ent = &tp->t_http_info[i];
4427                 /* Skip empty ones */
4428                 if (ent->flags == TCP_HTTP_TRACK_FLG_EMPTY)
4429                         continue;
4430                 /* Skip open ones */
4431                 if (ent->flags & TCP_HTTP_TRACK_FLG_OPEN)
4432                         continue;
4433                 if (SEQ_GEQ(ack_point, ent->end_seq)) {
4434                         /* We are past it -- free it */
4435                         tcp_http_log_req_info(tp, ent,
4436                                               i, TCP_HTTP_REQ_LOG_FREED, 0, 0);
4437                         tcp_http_free_a_slot(tp, ent);
4438                         ret++;
4439                 }
4440         }
4441         return (ret);
4442 }
4443
4444 int
4445 tcp_http_is_entry_comp(struct tcpcb *tp, struct http_sendfile_track *ent, tcp_seq ack_point)
4446 {
4447         if (tp->t_http_req == 0)
4448                 return(-1);
4449         if (tp->t_http_closed == 0)
4450                 return(-1);
4451         if (ent->flags == TCP_HTTP_TRACK_FLG_EMPTY)
4452                 return(-1);
4453         if (SEQ_GEQ(ack_point, ent->end_seq)) {
4454                 return (1);
4455         }
4456         return (0);
4457 }
4458
4459 struct http_sendfile_track *
4460 tcp_http_find_a_req_that_is_completed_by(struct tcpcb *tp, tcp_seq th_ack, int *ip)
4461 {
4462         /*
4463          * Given an ack point (th_ack) walk through our entries and
4464          * return the first one found that th_ack goes past the
4465          * end_seq.
4466          */
4467         struct http_sendfile_track *ent;
4468         int i;
4469
4470         if (tp->t_http_req == 0) {
4471                 /* none open */
4472                 return (NULL);
4473         }
4474         for(i = 0; i < MAX_TCP_HTTP_REQ; i++) {
4475                 ent = &tp->t_http_info[i];
4476                 if (ent->flags == TCP_HTTP_TRACK_FLG_EMPTY)
4477                         continue;
4478                 if ((ent->flags & TCP_HTTP_TRACK_FLG_OPEN) == 0) {
4479                         if (SEQ_GEQ(th_ack, ent->end_seq)) {
4480                                 *ip = i;
4481                                 return (ent);
4482                         }
4483                 }
4484         }
4485         return (NULL);
4486 }
4487
4488 struct http_sendfile_track *
4489 tcp_http_find_req_for_seq(struct tcpcb *tp, tcp_seq seq)
4490 {
4491         struct http_sendfile_track *ent;
4492         int i;
4493
4494         if (tp->t_http_req == 0) {
4495                 /* none open */
4496                 return (NULL);
4497         }
4498         for(i = 0; i < MAX_TCP_HTTP_REQ; i++) {
4499                 ent = &tp->t_http_info[i];
4500                 tcp_http_log_req_info(tp, ent, i, TCP_HTTP_REQ_LOG_SEARCH,
4501                                       (uint64_t)seq, 0);
4502                 if (ent->flags == TCP_HTTP_TRACK_FLG_EMPTY) {
4503                         continue;
4504                 }
4505                 if (ent->flags & TCP_HTTP_TRACK_FLG_OPEN) {
4506                         /*
4507                          * An open end request only needs to
4508                          * match the beginning seq or be
4509                          * all we have (once we keep going on
4510                          * a open end request we may have a seq
4511                          * wrap).
4512                          */
4513                         if ((SEQ_GEQ(seq, ent->start_seq)) ||
4514                             (tp->t_http_closed == 0))
4515                                 return (ent);
4516                 } else {
4517                         /*
4518                          * For this one we need to
4519                          * be a bit more careful if its
4520                          * completed at least.
4521                          */
4522                         if ((SEQ_GEQ(seq, ent->start_seq)) &&
4523                             (SEQ_LT(seq, ent->end_seq))) {
4524                                 return (ent);
4525                         }
4526                 }
4527         }
4528         return (NULL);
4529 }
4530
4531 /* Should this be in its own file tcp_http.c ? */
4532 struct http_sendfile_track *
4533 tcp_http_alloc_req_full(struct tcpcb *tp, struct http_req *req, uint64_t ts, int rec_dups)
4534 {
4535         struct http_sendfile_track *fil;
4536         int i, allocated;
4537
4538         /* In case the stack does not check for completions do so now */
4539         tcp_http_check_for_comp(tp, tp->snd_una);
4540         /* Check for stale entries */
4541         if (tp->t_http_req)
4542                 tcp_http_check_for_stale_entries(tp, ts,
4543                     (tp->t_http_req >= MAX_TCP_HTTP_REQ));
4544         /* Check to see if this is a duplicate of one not started */
4545         if (tp->t_http_req) {
4546                 for(i = 0, allocated = 0; i < MAX_TCP_HTTP_REQ; i++) {
4547                         fil = &tp->t_http_info[i];
4548                         if (fil->flags != TCP_HTTP_TRACK_FLG_USED)
4549                                 continue;
4550                         if ((fil->timestamp == req->timestamp) &&
4551                             (fil->start == req->start) &&
4552                             ((fil->flags & TCP_HTTP_TRACK_FLG_OPEN) ||
4553                              (fil->end == req->end))) {
4554                                 /*
4555                                  * We already have this request
4556                                  * and it has not been started with sendfile.
4557                                  * This probably means the user was returned
4558                                  * a 4xx of some sort and its going to age
4559                                  * out, lets not duplicate it.
4560                                  */
4561                                 return(fil);
4562                         }
4563                 }
4564         }
4565         /* Ok if there is no room at the inn we are in trouble */
4566         if (tp->t_http_req >= MAX_TCP_HTTP_REQ) {
4567                 tcp_trace_point(tp, TCP_TP_HTTP_LOG_FAIL);
4568                 for(i = 0; i < MAX_TCP_HTTP_REQ; i++) {
4569                         tcp_http_log_req_info(tp, &tp->t_http_info[i],
4570                             i, TCP_HTTP_REQ_LOG_ALLOCFAIL, 0, 0);
4571                 }
4572                 return (NULL);
4573         }
4574         for(i = 0, allocated = 0; i < MAX_TCP_HTTP_REQ; i++) {
4575                 fil = &tp->t_http_info[i];
4576                 if (fil->flags == TCP_HTTP_TRACK_FLG_EMPTY) {
4577                         allocated = 1;
4578                         fil->flags = TCP_HTTP_TRACK_FLG_USED;
4579                         fil->timestamp = req->timestamp;
4580                         fil->localtime = ts;
4581                         fil->start = req->start;
4582                         if (req->flags & TCP_LOG_HTTPD_RANGE_END) {
4583                                 fil->end = req->end;
4584                         } else {
4585                                 fil->end = 0;
4586                                 fil->flags |= TCP_HTTP_TRACK_FLG_OPEN;
4587                         }
4588                         /*
4589                          * We can set the min boundaries to the TCP Sequence space,
4590                          * but it might be found to be further up when sendfile
4591                          * actually runs on this range (if it ever does).
4592                          */
4593                         fil->sbcc_at_s = tptosocket(tp)->so_snd.sb_ccc;
4594                         fil->start_seq = tp->snd_una +
4595                             tptosocket(tp)->so_snd.sb_ccc;
4596                         fil->end_seq = (fil->start_seq + ((uint32_t)(fil->end - fil->start)));
4597                         if (tptosocket(tp)->so_snd.sb_tls_info) {
4598                                 /*
4599                                  * This session is doing TLS. Take a swag guess
4600                                  * at the overhead.
4601                                  */
4602                                 fil->end_seq += tcp_estimate_tls_overhead(
4603                                     tptosocket(tp), (fil->end - fil->start));
4604                         }
4605                         tp->t_http_req++;
4606                         if (fil->flags & TCP_HTTP_TRACK_FLG_OPEN)
4607                                 tp->t_http_open++;
4608                         else
4609                                 tp->t_http_closed++;
4610                         tcp_http_log_req_info(tp, fil, i,
4611                             TCP_HTTP_REQ_LOG_NEW, 0, 0);
4612                         break;
4613                 } else
4614                         fil = NULL;
4615         }
4616         return (fil);
4617 }
4618
4619 void
4620 tcp_http_alloc_req(struct tcpcb *tp, union tcp_log_userdata *user, uint64_t ts)
4621 {
4622         (void)tcp_http_alloc_req_full(tp, &user->http_req, ts, 1);
4623 }
4624 #endif
4625
4626 void
4627 tcp_log_socket_option(struct tcpcb *tp, uint32_t option_num, uint32_t option_val, int err)
4628 {
4629         if (tcp_bblogging_on(tp)) {
4630                 struct tcp_log_buffer *l;
4631
4632                 l = tcp_log_event(tp, NULL,
4633                         &tptosocket(tp)->so_rcv,
4634                         &tptosocket(tp)->so_snd,
4635                         TCP_LOG_SOCKET_OPT,
4636                         err, 0, NULL, 1,
4637                         NULL, NULL, 0, NULL);
4638                 if (l) {
4639                         l->tlb_flex1 = option_num;
4640                         l->tlb_flex2 = option_val;
4641                 }
4642         }
4643 }