sys/netinet/tcp_stacks/rack.c

   1 /*-
   2  * Copyright (c) 2016-2020 Netflix, Inc.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include "opt_inet.h"
  31 #include "opt_inet6.h"
  32 #include "opt_ipsec.h"
  33 #include "opt_tcpdebug.h"
  34 #include "opt_ratelimit.h"
  35 #include "opt_kern_tls.h"
  36 #include <sys/param.h>
  37 #include <sys/arb.h>
  38 #include <sys/module.h>
  39 #include <sys/kernel.h>
  40 #ifdef TCP_HHOOK
  41 #include <sys/hhook.h>
  42 #endif
  43 #include <sys/lock.h>
  44 #include <sys/malloc.h>
  45 #include <sys/lock.h>
  46 #include <sys/mutex.h>
  47 #include <sys/mbuf.h>
  48 #include <sys/proc.h>           /* for proc0 declaration */
  49 #include <sys/socket.h>
  50 #include <sys/socketvar.h>
  51 #include <sys/sysctl.h>
  52 #include <sys/systm.h>
  53 #ifdef STATS
  54 #include <sys/qmath.h>
  55 #include <sys/tree.h>
  56 #include <sys/stats.h> /* Must come after qmath.h and tree.h */
  57 #else
  58 #include <sys/tree.h>
  59 #endif
  60 #include <sys/refcount.h>
  61 #include <sys/queue.h>
  62 #include <sys/tim_filter.h>
  63 #include <sys/smp.h>
  64 #include <sys/kthread.h>
  65 #include <sys/kern_prefetch.h>
  66 #include <sys/protosw.h>
  67 #ifdef TCP_ACCOUNTING
  68 #include <sys/sched.h>
  69 #include <machine/cpu.h>
  70 #endif
  71 #include <vm/uma.h>
  72
  73 #include <net/route.h>
  74 #include <net/route/nhop.h>
  75 #include <net/vnet.h>
  76
  77 #define TCPSTATES               /* for logging */
  78
  79 #include <netinet/in.h>
  80 #include <netinet/in_kdtrace.h>
  81 #include <netinet/in_pcb.h>
  82 #include <netinet/ip.h>
  83 #include <netinet/ip_icmp.h>    /* required for icmp_var.h */
  84 #include <netinet/icmp_var.h>   /* for ICMP_BANDLIM */
  85 #include <netinet/ip_var.h>
  86 #include <netinet/ip6.h>
  87 #include <netinet6/in6_pcb.h>
  88 #include <netinet6/ip6_var.h>
  89 #include <netinet/tcp.h>
  90 #define TCPOUTFLAGS
  91 #include <netinet/tcp_fsm.h>
  92 #include <netinet/tcp_log_buf.h>
  93 #include <netinet/tcp_seq.h>
  94 #include <netinet/tcp_timer.h>
  95 #include <netinet/tcp_var.h>
  96 #include <netinet/tcp_syncache.h>
  97 #include <netinet/tcp_hpts.h>
  98 #include <netinet/tcp_ratelimit.h>
  99 #include <netinet/tcp_accounting.h>
 100 #include <netinet/tcpip.h>
 101 #include <netinet/cc/cc.h>
 102 #include <netinet/cc/cc_newreno.h>
 103 #include <netinet/tcp_fastopen.h>
 104 #include <netinet/tcp_lro.h>
 105 #ifdef NETFLIX_SHARED_CWND
 106 #include <netinet/tcp_shared_cwnd.h>
 107 #endif
 108 #ifdef TCPDEBUG
 109 #include <netinet/tcp_debug.h>
 110 #endif                          /* TCPDEBUG */
 111 #ifdef TCP_OFFLOAD
 112 #include <netinet/tcp_offload.h>
 113 #endif
 114 #ifdef INET6
 115 #include <netinet6/tcp6_var.h>
 116 #endif
 117 #include <netinet/tcp_ecn.h>
 118
 119 #include <netipsec/ipsec_support.h>
 120
 121 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
 122 #include <netipsec/ipsec.h>
 123 #include <netipsec/ipsec6.h>
 124 #endif                          /* IPSEC */
 125
 126 #include <netinet/udp.h>
 127 #include <netinet/udp_var.h>
 128 #include <machine/in_cksum.h>
 129
 130 #ifdef MAC
 131 #include <security/mac/mac_framework.h>
 132 #endif
 133 #include "sack_filter.h"
 134 #include "tcp_rack.h"
 135 #include "rack_bbr_common.h"
 136
 137 uma_zone_t rack_zone;
 138 uma_zone_t rack_pcb_zone;
 139
 140 #ifndef TICKS2SBT
 141 #define TICKS2SBT(__t)  (tick_sbt * ((sbintime_t)(__t)))
 142 #endif
 143
 144 VNET_DECLARE(uint32_t, newreno_beta);
 145 VNET_DECLARE(uint32_t, newreno_beta_ecn);
 146 #define V_newreno_beta VNET(newreno_beta)
 147 #define V_newreno_beta_ecn VNET(newreno_beta_ecn)
 148
 149
 150 MALLOC_DEFINE(M_TCPFSB, "tcp_fsb", "TCP fast send block");
 151 MALLOC_DEFINE(M_TCPDO, "tcp_do", "TCP deferred options");
 152
 153 struct sysctl_ctx_list rack_sysctl_ctx;
 154 struct sysctl_oid *rack_sysctl_root;
 155
 156 #define CUM_ACKED 1
 157 #define SACKED 2
 158
 159 /*
 160  * The RACK module incorporates a number of
 161  * TCP ideas that have been put out into the IETF
 162  * over the last few years:
 163  * - Matt Mathis's Rate Halving which slowly drops
 164  *    the congestion window so that the ack clock can
 165  *    be maintained during a recovery.
 166  * - Yuchung Cheng's RACK TCP (for which its named) that
 167  *    will stop us using the number of dup acks and instead
 168  *    use time as the gage of when we retransmit.
 169  * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
 170  *    of Dukkipati et.al.
 171  * RACK depends on SACK, so if an endpoint arrives that
 172  * cannot do SACK the state machine below will shuttle the
 173  * connection back to using the "default" TCP stack that is
 174  * in FreeBSD.
 175  *
 176  * To implement RACK the original TCP stack was first decomposed
 177  * into a functional state machine with individual states
 178  * for each of the possible TCP connection states. The do_segment
 179  * functions role in life is to mandate the connection supports SACK
 180  * initially and then assure that the RACK state matches the conenction
 181  * state before calling the states do_segment function. Each
 182  * state is simplified due to the fact that the original do_segment
 183  * has been decomposed and we *know* what state we are in (no
 184  * switches on the state) and all tests for SACK are gone. This
 185  * greatly simplifies what each state does.
 186  *
 187  * TCP output is also over-written with a new version since it
 188  * must maintain the new rack scoreboard.
 189  *
 190  */
 191 static int32_t rack_tlp_thresh = 1;
 192 static int32_t rack_tlp_limit = 2;      /* No more than 2 TLPs w-out new data */
 193 static int32_t rack_tlp_use_greater = 1;
 194 static int32_t rack_reorder_thresh = 2;
 195 static int32_t rack_reorder_fade = 60000000;    /* 0 - never fade, def 60,000,000
 196                                                  * - 60 seconds */
 197 static uint8_t rack_req_measurements = 1;
 198 /* Attack threshold detections */
 199 static uint32_t rack_highest_sack_thresh_seen = 0;
 200 static uint32_t rack_highest_move_thresh_seen = 0;
 201 static int32_t rack_enable_hw_pacing = 0; /* Due to CCSP keep it off by default */
 202 static int32_t rack_hw_pace_extra_slots = 2;    /* 2 extra MSS time betweens */
 203 static int32_t rack_hw_rate_caps = 1; /* 1; */
 204 static int32_t rack_hw_rate_min = 0; /* 1500000;*/
 205 static int32_t rack_hw_rate_to_low = 0; /* 1200000; */
 206 static int32_t rack_hw_up_only = 1;
 207 static int32_t rack_stats_gets_ms_rtt = 1;
 208 static int32_t rack_prr_addbackmax = 2;
 209 static int32_t rack_do_hystart = 0;
 210 static int32_t rack_apply_rtt_with_reduced_conf = 0;
 211
 212 static int32_t rack_pkt_delay = 1000;
 213 static int32_t rack_send_a_lot_in_prr = 1;
 214 static int32_t rack_min_to = 1000;      /* Number of microsecond  min timeout */
 215 static int32_t rack_verbose_logging = 0;
 216 static int32_t rack_ignore_data_after_close = 1;
 217 static int32_t rack_enable_shared_cwnd = 1;
 218 static int32_t rack_use_cmp_acks = 1;
 219 static int32_t rack_use_fsb = 1;
 220 static int32_t rack_use_rfo = 1;
 221 static int32_t rack_use_rsm_rfo = 1;
 222 static int32_t rack_max_abc_post_recovery = 2;
 223 static int32_t rack_client_low_buf = 0;
 224 static int32_t rack_dsack_std_based = 0x3;      /* bit field bit 1 sets rc_rack_tmr_std_based and bit 2 sets rc_rack_use_dsack */
 225 #ifdef TCP_ACCOUNTING
 226 static int32_t rack_tcp_accounting = 0;
 227 #endif
 228 static int32_t rack_limits_scwnd = 1;
 229 static int32_t rack_enable_mqueue_for_nonpaced = 0;
 230 static int32_t rack_disable_prr = 0;
 231 static int32_t use_rack_rr = 1;
 232 static int32_t rack_non_rxt_use_cr = 0; /* does a non-rxt in recovery use the configured rate (ss/ca)? */
 233 static int32_t rack_persist_min = 250000;       /* 250usec */
 234 static int32_t rack_persist_max = 2000000;      /* 2 Second in usec's */
 235 static int32_t rack_sack_not_required = 1;      /* set to one to allow non-sack to use rack */
 236 static int32_t rack_default_init_window = 0;    /* Use system default */
 237 static int32_t rack_limit_time_with_srtt = 0;
 238 static int32_t rack_autosndbuf_inc = 20;        /* In percentage form */
 239 static int32_t rack_enobuf_hw_boost_mult = 2;   /* How many times the hw rate we boost slot using time_between */
 240 static int32_t rack_enobuf_hw_max = 12000;      /* 12 ms in usecs */
 241 static int32_t rack_enobuf_hw_min = 10000;      /* 10 ms in usecs */
 242 static int32_t rack_hw_rwnd_factor = 2;         /* How many max_segs the rwnd must be before we hold off sending */
 243
 244 /*
 245  * Currently regular tcp has a rto_min of 30ms
 246  * the backoff goes 12 times so that ends up
 247  * being a total of 122.850 seconds before a
 248  * connection is killed.
 249  */
 250 static uint32_t rack_def_data_window = 20;
 251 static uint32_t rack_goal_bdp = 2;
 252 static uint32_t rack_min_srtts = 1;
 253 static uint32_t rack_min_measure_usec = 0;
 254 static int32_t rack_tlp_min = 10000;    /* 10ms */
 255 static int32_t rack_rto_min = 30000;    /* 30,000 usec same as main freebsd */
 256 static int32_t rack_rto_max = 4000000;  /* 4 seconds in usec's */
 257 static const int32_t rack_free_cache = 2;
 258 static int32_t rack_hptsi_segments = 40;
 259 static int32_t rack_rate_sample_method = USE_RTT_LOW;
 260 static int32_t rack_pace_every_seg = 0;
 261 static int32_t rack_delayed_ack_time = 40000;   /* 40ms in usecs */
 262 static int32_t rack_slot_reduction = 4;
 263 static int32_t rack_wma_divisor = 8;            /* For WMA calculation */
 264 static int32_t rack_cwnd_block_ends_measure = 0;
 265 static int32_t rack_rwnd_block_ends_measure = 0;
 266 static int32_t rack_def_profile = 0;
 267
 268 static int32_t rack_lower_cwnd_at_tlp = 0;
 269 static int32_t rack_limited_retran = 0;
 270 static int32_t rack_always_send_oldest = 0;
 271 static int32_t rack_tlp_threshold_use = TLP_USE_TWO_ONE;
 272
 273 static uint16_t rack_per_of_gp_ss = 250;        /* 250 % slow-start */
 274 static uint16_t rack_per_of_gp_ca = 200;        /* 200 % congestion-avoidance */
 275 static uint16_t rack_per_of_gp_rec = 200;       /* 200 % of bw */
 276
 277 /* Probertt */
 278 static uint16_t rack_per_of_gp_probertt = 60;   /* 60% of bw */
 279 static uint16_t rack_per_of_gp_lowthresh = 40;  /* 40% is bottom */
 280 static uint16_t rack_per_of_gp_probertt_reduce = 10; /* 10% reduction */
 281 static uint16_t rack_atexit_prtt_hbp = 130;     /* Clamp to 130% on exit prtt if highly buffered path */
 282 static uint16_t rack_atexit_prtt = 130; /* Clamp to 100% on exit prtt if non highly buffered path */
 283
 284 static uint32_t rack_max_drain_wait = 2;        /* How man gp srtt's before we give up draining */
 285 static uint32_t rack_must_drain = 1;            /* How many GP srtt's we *must* wait */
 286 static uint32_t rack_probertt_use_min_rtt_entry = 1;    /* Use the min to calculate the goal else gp_srtt */
 287 static uint32_t rack_probertt_use_min_rtt_exit = 0;
 288 static uint32_t rack_probe_rtt_sets_cwnd = 0;
 289 static uint32_t rack_probe_rtt_safety_val = 2000000;    /* No more than 2 sec in probe-rtt */
 290 static uint32_t rack_time_between_probertt = 9600000;   /* 9.6 sec in usecs */
 291 static uint32_t rack_probertt_gpsrtt_cnt_mul = 0;       /* How many srtt periods does probe-rtt last top fraction */
 292 static uint32_t rack_probertt_gpsrtt_cnt_div = 0;       /* How many srtt periods does probe-rtt last bottom fraction */
 293 static uint32_t rack_min_probertt_hold = 40000;         /* Equal to delayed ack time */
 294 static uint32_t rack_probertt_filter_life = 10000000;
 295 static uint32_t rack_probertt_lower_within = 10;
 296 static uint32_t rack_min_rtt_movement = 250000; /* Must move at least 250ms (in microseconds)  to count as a lowering */
 297 static int32_t rack_pace_one_seg = 0;           /* Shall we pace for less than 1.4Meg 1MSS at a time */
 298 static int32_t rack_probertt_clear_is = 1;
 299 static int32_t rack_max_drain_hbp = 1;          /* Extra drain times gpsrtt for highly buffered paths */
 300 static int32_t rack_hbp_thresh = 3;             /* what is the divisor max_rtt/min_rtt to decided a hbp */
 301
 302 /* Part of pacing */
 303 static int32_t rack_max_per_above = 30;         /* When we go to increment stop if above 100+this% */
 304
 305 /* Timely information */
 306 /* Combine these two gives the range of 'no change' to bw */
 307 /* ie the up/down provide the upper and lower bound */
 308 static int32_t rack_gp_per_bw_mul_up = 2;       /* 2% */
 309 static int32_t rack_gp_per_bw_mul_down = 4;     /* 4% */
 310 static int32_t rack_gp_rtt_maxmul = 3;          /* 3 x maxmin */
 311 static int32_t rack_gp_rtt_minmul = 1;          /* minrtt + (minrtt/mindiv) is lower rtt */
 312 static int32_t rack_gp_rtt_mindiv = 4;          /* minrtt + (minrtt * minmul/mindiv) is lower rtt */
 313 static int32_t rack_gp_decrease_per = 20;       /* 20% decrease in multiplier */
 314 static int32_t rack_gp_increase_per = 2;        /* 2% increase in multiplier */
 315 static int32_t rack_per_lower_bound = 50;       /* Don't allow to drop below this multiplier */
 316 static int32_t rack_per_upper_bound_ss = 0;     /* Don't allow SS to grow above this */
 317 static int32_t rack_per_upper_bound_ca = 0;     /* Don't allow CA to grow above this */
 318 static int32_t rack_do_dyn_mul = 0;             /* Are the rack gp multipliers dynamic */
 319 static int32_t rack_gp_no_rec_chg = 1;          /* Prohibit recovery from reducing it's multiplier */
 320 static int32_t rack_timely_dec_clear = 6;       /* Do we clear decrement count at a value (6)? */
 321 static int32_t rack_timely_max_push_rise = 3;   /* One round of pushing */
 322 static int32_t rack_timely_max_push_drop = 3;   /* Three round of pushing */
 323 static int32_t rack_timely_min_segs = 4;        /* 4 segment minimum */
 324 static int32_t rack_use_max_for_nobackoff = 0;
 325 static int32_t rack_timely_int_timely_only = 0; /* do interim timely's only use the timely algo (no b/w changes)? */
 326 static int32_t rack_timely_no_stopping = 0;
 327 static int32_t rack_down_raise_thresh = 100;
 328 static int32_t rack_req_segs = 1;
 329 static uint64_t rack_bw_rate_cap = 0;
 330 static uint32_t rack_trace_point_config = 0;
 331 static uint32_t rack_trace_point_bb_mode = 4;
 332 static int32_t rack_trace_point_count = 0;
 333
 334
 335 /* Weird delayed ack mode */
 336 static int32_t rack_use_imac_dack = 0;
 337 /* Rack specific counters */
 338 counter_u64_t rack_saw_enobuf;
 339 counter_u64_t rack_saw_enobuf_hw;
 340 counter_u64_t rack_saw_enetunreach;
 341 counter_u64_t rack_persists_sends;
 342 counter_u64_t rack_persists_acks;
 343 counter_u64_t rack_persists_loss;
 344 counter_u64_t rack_persists_lost_ends;
 345 #ifdef INVARIANTS
 346 counter_u64_t rack_adjust_map_bw;
 347 #endif
 348 /* Tail loss probe counters */
 349 counter_u64_t rack_tlp_tot;
 350 counter_u64_t rack_tlp_newdata;
 351 counter_u64_t rack_tlp_retran;
 352 counter_u64_t rack_tlp_retran_bytes;
 353 counter_u64_t rack_to_tot;
 354 counter_u64_t rack_hot_alloc;
 355 counter_u64_t rack_to_alloc;
 356 counter_u64_t rack_to_alloc_hard;
 357 counter_u64_t rack_to_alloc_emerg;
 358 counter_u64_t rack_to_alloc_limited;
 359 counter_u64_t rack_alloc_limited_conns;
 360 counter_u64_t rack_split_limited;
 361
 362 counter_u64_t rack_multi_single_eq;
 363 counter_u64_t rack_proc_non_comp_ack;
 364
 365 counter_u64_t rack_fto_send;
 366 counter_u64_t rack_fto_rsm_send;
 367 counter_u64_t rack_nfto_resend;
 368 counter_u64_t rack_non_fto_send;
 369 counter_u64_t rack_extended_rfo;
 370
 371 counter_u64_t rack_sack_proc_all;
 372 counter_u64_t rack_sack_proc_short;
 373 counter_u64_t rack_sack_proc_restart;
 374 counter_u64_t rack_sack_attacks_detected;
 375 counter_u64_t rack_sack_attacks_reversed;
 376 counter_u64_t rack_sack_used_next_merge;
 377 counter_u64_t rack_sack_splits;
 378 counter_u64_t rack_sack_used_prev_merge;
 379 counter_u64_t rack_sack_skipped_acked;
 380 counter_u64_t rack_ack_total;
 381 counter_u64_t rack_express_sack;
 382 counter_u64_t rack_sack_total;
 383 counter_u64_t rack_move_none;
 384 counter_u64_t rack_move_some;
 385
 386 counter_u64_t rack_input_idle_reduces;
 387 counter_u64_t rack_collapsed_win;
 388 counter_u64_t rack_try_scwnd;
 389 counter_u64_t rack_hw_pace_init_fail;
 390 counter_u64_t rack_hw_pace_lost;
 391
 392 counter_u64_t rack_out_size[TCP_MSS_ACCT_SIZE];
 393 counter_u64_t rack_opts_arry[RACK_OPTS_SIZE];
 394
 395
 396 #define RACK_REXMTVAL(tp) max(rack_rto_min, ((tp)->t_srtt + ((tp)->t_rttvar << 2)))
 397
 398 #define RACK_TCPT_RANGESET(tv, value, tvmin, tvmax, slop) do {  \
 399         (tv) = (value) + slop;   \
 400         if ((u_long)(tv) < (u_long)(tvmin)) \
 401                 (tv) = (tvmin); \
 402         if ((u_long)(tv) > (u_long)(tvmax)) \
 403                 (tv) = (tvmax); \
 404 } while (0)
 405
 406 static void
 407 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick,  int event, int line);
 408
 409 static int
 410 rack_process_ack(struct mbuf *m, struct tcphdr *th,
 411     struct socket *so, struct tcpcb *tp, struct tcpopt *to,
 412     uint32_t tiwin, int32_t tlen, int32_t * ofia, int32_t thflags, int32_t * ret_val);
 413 static int
 414 rack_process_data(struct mbuf *m, struct tcphdr *th,
 415     struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
 416     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt);
 417 static void
 418 rack_ack_received(struct tcpcb *tp, struct tcp_rack *rack,
 419    uint32_t th_ack, uint16_t nsegs, uint16_t type, int32_t recovery);
 420 static struct rack_sendmap *rack_alloc(struct tcp_rack *rack);
 421 static struct rack_sendmap *rack_alloc_limit(struct tcp_rack *rack,
 422     uint8_t limit_type);
 423 static struct rack_sendmap *
 424 rack_check_recovery_mode(struct tcpcb *tp,
 425     uint32_t tsused);
 426 static void
 427 rack_cong_signal(struct tcpcb *tp,
 428                  uint32_t type, uint32_t ack, int );
 429 static void rack_counter_destroy(void);
 430 static int
 431 rack_ctloutput(struct inpcb *inp, struct sockopt *sopt);
 432 static int32_t rack_ctor(void *mem, int32_t size, void *arg, int32_t how);
 433 static void
 434 rack_set_pace_segments(struct tcpcb *tp, struct tcp_rack *rack, uint32_t line, uint64_t *fill_override);
 435 static void
 436 rack_do_segment(struct mbuf *m, struct tcphdr *th,
 437     struct socket *so, struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
 438     uint8_t iptos);
 439 static void rack_dtor(void *mem, int32_t size, void *arg);
 440 static void
 441 rack_log_alt_to_to_cancel(struct tcp_rack *rack,
 442     uint32_t flex1, uint32_t flex2,
 443     uint32_t flex3, uint32_t flex4,
 444     uint32_t flex5, uint32_t flex6,
 445     uint16_t flex7, uint8_t mod);
 446
 447 static void
 448 rack_log_pacing_delay_calc(struct tcp_rack *rack, uint32_t len, uint32_t slot,
 449    uint64_t bw_est, uint64_t bw, uint64_t len_time, int method, int line,
 450    struct rack_sendmap *rsm, uint8_t quality);
 451 static struct rack_sendmap *
 452 rack_find_high_nonack(struct tcp_rack *rack,
 453     struct rack_sendmap *rsm);
 454 static struct rack_sendmap *rack_find_lowest_rsm(struct tcp_rack *rack);
 455 static void rack_free(struct tcp_rack *rack, struct rack_sendmap *rsm);
 456 static void rack_fini(struct tcpcb *tp, int32_t tcb_is_purged);
 457 static int rack_get_sockopt(struct inpcb *inp, struct sockopt *sopt);
 458 static void
 459 rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack,
 460                             tcp_seq th_ack, int line, uint8_t quality);
 461 static uint32_t
 462 rack_get_pacing_len(struct tcp_rack *rack, uint64_t bw, uint32_t mss);
 463 static int32_t rack_handoff_ok(struct tcpcb *tp);
 464 static int32_t rack_init(struct tcpcb *tp);
 465 static void rack_init_sysctls(void);
 466 static void
 467 rack_log_ack(struct tcpcb *tp, struct tcpopt *to,
 468     struct tcphdr *th, int entered_rec, int dup_ack_struck);
 469 static void
 470 rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len,
 471     uint32_t seq_out, uint16_t th_flags, int32_t err, uint64_t ts,
 472     struct rack_sendmap *hintrsm, uint16_t add_flags, struct mbuf *s_mb, uint32_t s_moff, int hw_tls);
 473
 474 static void
 475 rack_log_sack_passed(struct tcpcb *tp, struct tcp_rack *rack,
 476     struct rack_sendmap *rsm);
 477 static void rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rsm);
 478 static int32_t rack_output(struct tcpcb *tp);
 479
 480 static uint32_t
 481 rack_proc_sack_blk(struct tcpcb *tp, struct tcp_rack *rack,
 482     struct sackblk *sack, struct tcpopt *to, struct rack_sendmap **prsm,
 483     uint32_t cts, int *moved_two);
 484 static void rack_post_recovery(struct tcpcb *tp, uint32_t th_seq);
 485 static void rack_remxt_tmr(struct tcpcb *tp);
 486 static int rack_set_sockopt(struct inpcb *inp, struct sockopt *sopt);
 487 static void rack_set_state(struct tcpcb *tp, struct tcp_rack *rack);
 488 static int32_t rack_stopall(struct tcpcb *tp);
 489 static void
 490 rack_timer_activate(struct tcpcb *tp, uint32_t timer_type,
 491     uint32_t delta);
 492 static int32_t rack_timer_active(struct tcpcb *tp, uint32_t timer_type);
 493 static void rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int line);
 494 static void rack_timer_stop(struct tcpcb *tp, uint32_t timer_type);
 495 static uint32_t
 496 rack_update_entry(struct tcpcb *tp, struct tcp_rack *rack,
 497     struct rack_sendmap *rsm, uint64_t ts, int32_t * lenp, uint16_t add_flag);
 498 static void
 499 rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack,
 500     struct rack_sendmap *rsm, uint64_t ts, uint16_t add_flag);
 501 static int
 502 rack_update_rtt(struct tcpcb *tp, struct tcp_rack *rack,
 503     struct rack_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, tcp_seq th_ack);
 504 static int32_t tcp_addrack(module_t mod, int32_t type, void *data);
 505 static int
 506 rack_do_close_wait(struct mbuf *m, struct tcphdr *th,
 507     struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
 508     int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
 509 static int
 510 rack_do_closing(struct mbuf *m, struct tcphdr *th,
 511     struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
 512     int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
 513 static int
 514 rack_do_established(struct mbuf *m, struct tcphdr *th,
 515     struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
 516     int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
 517 static int
 518 rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th,
 519     struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
 520     int32_t tlen, uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos);
 521 static int
 522 rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th,
 523     struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
 524     int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
 525 static int
 526 rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th,
 527     struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
 528     int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
 529 static int
 530 rack_do_lastack(struct mbuf *m, struct tcphdr *th,
 531     struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
 532     int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
 533 static int
 534 rack_do_syn_recv(struct mbuf *m, struct tcphdr *th,
 535     struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
 536     int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
 537 static int
 538 rack_do_syn_sent(struct mbuf *m, struct tcphdr *th,
 539     struct socket *so, struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen,
 540     int32_t tlen, uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos);
 541 struct rack_sendmap *
 542 tcp_rack_output(struct tcpcb *tp, struct tcp_rack *rack,
 543     uint32_t tsused);
 544 static void tcp_rack_xmit_timer(struct tcp_rack *rack, int32_t rtt,
 545     uint32_t len, uint32_t us_tim, int confidence, struct rack_sendmap *rsm, uint16_t rtrcnt);
 546 static void
 547      tcp_rack_partialack(struct tcpcb *tp);
 548 static int
 549 rack_set_profile(struct tcp_rack *rack, int prof);
 550 static void
 551 rack_apply_deferred_options(struct tcp_rack *rack);
 552
 553 int32_t rack_clear_counter=0;
 554
 555 static inline void
 556 rack_trace_point(struct tcp_rack *rack, int num)
 557 {
 558         if (((rack_trace_point_config == num)  ||
 559              (rack_trace_point_config = 0xffffffff)) &&
 560             (rack_trace_point_bb_mode != 0) &&
 561             (rack_trace_point_count > 0) &&
 562             (rack->rc_tp->t_logstate == 0)) {
 563                 int res;
 564                 res = atomic_fetchadd_int(&rack_trace_point_count, -1);
 565                 if (res > 0) {
 566                         rack->rc_tp->t_logstate = rack_trace_point_bb_mode;
 567                 } else {
 568                         /* Loss a race assure its zero now */
 569                         rack_trace_point_count = 0;
 570                 }
 571         }
 572 }
 573
 574 static void
 575 rack_set_cc_pacing(struct tcp_rack *rack)
 576 {
 577         struct sockopt sopt;
 578         struct cc_newreno_opts opt;
 579         struct newreno old, *ptr;
 580         struct tcpcb *tp;
 581         int error;
 582
 583         if (rack->rc_pacing_cc_set)
 584                 return;
 585
 586         tp = rack->rc_tp;
 587         if (tp->cc_algo == NULL) {
 588                 /* Tcb is leaving */
 589                 return;
 590         }
 591         rack->rc_pacing_cc_set = 1;
 592         if (strcmp(tp->cc_algo->name, CCALGONAME_NEWRENO) != 0) {
 593                 /* Not new-reno we can't play games with beta! */
 594                 goto out;
 595         }
 596         ptr = ((struct newreno *)tp->ccv->cc_data);
 597         if (CC_ALGO(tp)->ctl_output == NULL)  {
 598                 /* Huh, why does new_reno no longer have a set function? */
 599                 goto out;
 600         }
 601         if (ptr == NULL) {
 602                 /* Just the default values */
 603                 old.beta = V_newreno_beta_ecn;
 604                 old.beta_ecn = V_newreno_beta_ecn;
 605                 old.newreno_flags = 0;
 606         } else {
 607                 old.beta = ptr->beta;
 608                 old.beta_ecn = ptr->beta_ecn;
 609                 old.newreno_flags = ptr->newreno_flags;
 610         }
 611         sopt.sopt_valsize = sizeof(struct cc_newreno_opts);
 612         sopt.sopt_dir = SOPT_SET;
 613         opt.name = CC_NEWRENO_BETA;
 614         opt.val = rack->r_ctl.rc_saved_beta.beta;
 615         error = CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt);
 616         if (error)  {
 617                 goto out;
 618         }
 619         /*
 620          * Hack alert we need to set in our newreno_flags
 621          * so that Abe behavior is also applied.
 622          */
 623         ((struct newreno *)tp->ccv->cc_data)->newreno_flags |= CC_NEWRENO_BETA_ECN_ENABLED;
 624         opt.name = CC_NEWRENO_BETA_ECN;
 625         opt.val = rack->r_ctl.rc_saved_beta.beta_ecn;
 626         error = CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt);
 627         if (error) {
 628                 goto out;
 629         }
 630         /* Save off the original values for restoral */
 631         memcpy(&rack->r_ctl.rc_saved_beta, &old, sizeof(struct newreno));
 632 out:
 633         if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
 634                 union tcp_log_stackspecific log;
 635                 struct timeval tv;
 636
 637                 ptr = ((struct newreno *)tp->ccv->cc_data);
 638                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
 639                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
 640                 if (ptr) {
 641                         log.u_bbr.flex1 = ptr->beta;
 642                         log.u_bbr.flex2 = ptr->beta_ecn;
 643                         log.u_bbr.flex3 = ptr->newreno_flags;
 644                 }
 645                 log.u_bbr.flex4 = rack->r_ctl.rc_saved_beta.beta;
 646                 log.u_bbr.flex5 = rack->r_ctl.rc_saved_beta.beta_ecn;
 647                 log.u_bbr.flex6 = rack->r_ctl.rc_saved_beta.newreno_flags;
 648                 log.u_bbr.flex7 = rack->gp_ready;
 649                 log.u_bbr.flex7 <<= 1;
 650                 log.u_bbr.flex7 |= rack->use_fixed_rate;
 651                 log.u_bbr.flex7 <<= 1;
 652                 log.u_bbr.flex7 |= rack->rc_pacing_cc_set;
 653                 log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
 654                 log.u_bbr.flex8 = 3;
 655                 tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, error,
 656                                0, &log, false, NULL, NULL, 0, &tv);
 657         }
 658 }
 659
 660 static void
 661 rack_undo_cc_pacing(struct tcp_rack *rack)
 662 {
 663         struct newreno old, *ptr;
 664         struct tcpcb *tp;
 665
 666         if (rack->rc_pacing_cc_set == 0)
 667                 return;
 668         tp = rack->rc_tp;
 669         rack->rc_pacing_cc_set = 0;
 670         if (tp->cc_algo == NULL)
 671                 /* Tcb is leaving */
 672                 return;
 673         if (strcmp(tp->cc_algo->name, CCALGONAME_NEWRENO) != 0) {
 674                 /* Not new-reno nothing to do! */
 675                 return;
 676         }
 677         ptr = ((struct newreno *)tp->ccv->cc_data);
 678         if (ptr == NULL) {
 679                 /*
 680                  * This happens at rack_fini() if the
 681                  * cc module gets freed on us. In that
 682                  * case we loose our "new" settings but
 683                  * thats ok, since the tcb is going away anyway.
 684                  */
 685                 return;
 686         }
 687         /* Grab out our set values */
 688         memcpy(&old, ptr, sizeof(struct newreno));
 689         /* Copy back in the original values */
 690         memcpy(ptr, &rack->r_ctl.rc_saved_beta, sizeof(struct newreno));
 691         /* Now save back the values we had set in (for when pacing is restored) */
 692         memcpy(&rack->r_ctl.rc_saved_beta, &old, sizeof(struct newreno));
 693         if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
 694                 union tcp_log_stackspecific log;
 695                 struct timeval tv;
 696
 697                 ptr = ((struct newreno *)tp->ccv->cc_data);
 698                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
 699                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
 700                 log.u_bbr.flex1 = ptr->beta;
 701                 log.u_bbr.flex2 = ptr->beta_ecn;
 702                 log.u_bbr.flex3 = ptr->newreno_flags;
 703                 log.u_bbr.flex4 = rack->r_ctl.rc_saved_beta.beta;
 704                 log.u_bbr.flex5 = rack->r_ctl.rc_saved_beta.beta_ecn;
 705                 log.u_bbr.flex6 = rack->r_ctl.rc_saved_beta.newreno_flags;
 706                 log.u_bbr.flex7 = rack->gp_ready;
 707                 log.u_bbr.flex7 <<= 1;
 708                 log.u_bbr.flex7 |= rack->use_fixed_rate;
 709                 log.u_bbr.flex7 <<= 1;
 710                 log.u_bbr.flex7 |= rack->rc_pacing_cc_set;
 711                 log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
 712                 log.u_bbr.flex8 = 4;
 713                 tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
 714                                0, &log, false, NULL, NULL, 0, &tv);
 715         }
 716 }
 717
 718 #ifdef NETFLIX_PEAKRATE
 719 static inline void
 720 rack_update_peakrate_thr(struct tcpcb *tp)
 721 {
 722         /* Keep in mind that t_maxpeakrate is in B/s. */
 723         uint64_t peak;
 724         peak = uqmax((tp->t_maxseg * 2),
 725                      (((uint64_t)tp->t_maxpeakrate * (uint64_t)(tp->t_srtt)) / (uint64_t)HPTS_USEC_IN_SEC));
 726         tp->t_peakrate_thr = (uint32_t)uqmin(peak, UINT32_MAX);
 727 }
 728 #endif
 729
 730 static int
 731 sysctl_rack_clear(SYSCTL_HANDLER_ARGS)
 732 {
 733         uint32_t stat;
 734         int32_t error;
 735
 736         error = SYSCTL_OUT(req, &rack_clear_counter, sizeof(uint32_t));
 737         if (error || req->newptr == NULL)
 738                 return error;
 739
 740         error = SYSCTL_IN(req, &stat, sizeof(uint32_t));
 741         if (error)
 742                 return (error);
 743         if (stat == 1) {
 744 #ifdef INVARIANTS
 745                 printf("Clearing RACK counters\n");
 746 #endif
 747                 counter_u64_zero(rack_tlp_tot);
 748                 counter_u64_zero(rack_tlp_newdata);
 749                 counter_u64_zero(rack_tlp_retran);
 750                 counter_u64_zero(rack_tlp_retran_bytes);
 751                 counter_u64_zero(rack_to_tot);
 752                 counter_u64_zero(rack_saw_enobuf);
 753                 counter_u64_zero(rack_saw_enobuf_hw);
 754                 counter_u64_zero(rack_saw_enetunreach);
 755                 counter_u64_zero(rack_persists_sends);
 756                 counter_u64_zero(rack_persists_acks);
 757                 counter_u64_zero(rack_persists_loss);
 758                 counter_u64_zero(rack_persists_lost_ends);
 759 #ifdef INVARIANTS
 760                 counter_u64_zero(rack_adjust_map_bw);
 761 #endif
 762                 counter_u64_zero(rack_to_alloc_hard);
 763                 counter_u64_zero(rack_to_alloc_emerg);
 764                 counter_u64_zero(rack_sack_proc_all);
 765                 counter_u64_zero(rack_fto_send);
 766                 counter_u64_zero(rack_fto_rsm_send);
 767                 counter_u64_zero(rack_extended_rfo);
 768                 counter_u64_zero(rack_hw_pace_init_fail);
 769                 counter_u64_zero(rack_hw_pace_lost);
 770                 counter_u64_zero(rack_non_fto_send);
 771                 counter_u64_zero(rack_nfto_resend);
 772                 counter_u64_zero(rack_sack_proc_short);
 773                 counter_u64_zero(rack_sack_proc_restart);
 774                 counter_u64_zero(rack_to_alloc);
 775                 counter_u64_zero(rack_to_alloc_limited);
 776                 counter_u64_zero(rack_alloc_limited_conns);
 777                 counter_u64_zero(rack_split_limited);
 778                 counter_u64_zero(rack_multi_single_eq);
 779                 counter_u64_zero(rack_proc_non_comp_ack);
 780                 counter_u64_zero(rack_sack_attacks_detected);
 781                 counter_u64_zero(rack_sack_attacks_reversed);
 782                 counter_u64_zero(rack_sack_used_next_merge);
 783                 counter_u64_zero(rack_sack_used_prev_merge);
 784                 counter_u64_zero(rack_sack_splits);
 785                 counter_u64_zero(rack_sack_skipped_acked);
 786                 counter_u64_zero(rack_ack_total);
 787                 counter_u64_zero(rack_express_sack);
 788                 counter_u64_zero(rack_sack_total);
 789                 counter_u64_zero(rack_move_none);
 790                 counter_u64_zero(rack_move_some);
 791                 counter_u64_zero(rack_try_scwnd);
 792                 counter_u64_zero(rack_collapsed_win);
 793         }
 794         rack_clear_counter = 0;
 795         return (0);
 796 }
 797
 798 static void
 799 rack_init_sysctls(void)
 800 {
 801         struct sysctl_oid *rack_counters;
 802         struct sysctl_oid *rack_attack;
 803         struct sysctl_oid *rack_pacing;
 804         struct sysctl_oid *rack_timely;
 805         struct sysctl_oid *rack_timers;
 806         struct sysctl_oid *rack_tlp;
 807         struct sysctl_oid *rack_misc;
 808         struct sysctl_oid *rack_features;
 809         struct sysctl_oid *rack_measure;
 810         struct sysctl_oid *rack_probertt;
 811         struct sysctl_oid *rack_hw_pacing;
 812         struct sysctl_oid *rack_tracepoint;
 813
 814         rack_attack = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
 815             SYSCTL_CHILDREN(rack_sysctl_root),
 816             OID_AUTO,
 817             "sack_attack",
 818             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 819             "Rack Sack Attack Counters and Controls");
 820         rack_counters = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
 821             SYSCTL_CHILDREN(rack_sysctl_root),
 822             OID_AUTO,
 823             "stats",
 824             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 825             "Rack Counters");
 826         SYSCTL_ADD_S32(&rack_sysctl_ctx,
 827             SYSCTL_CHILDREN(rack_sysctl_root),
 828             OID_AUTO, "rate_sample_method", CTLFLAG_RW,
 829             &rack_rate_sample_method , USE_RTT_LOW,
 830             "What method should we use for rate sampling 0=high, 1=low ");
 831         /* Probe rtt related controls */
 832         rack_probertt = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
 833             SYSCTL_CHILDREN(rack_sysctl_root),
 834             OID_AUTO,
 835             "probertt",
 836             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 837             "ProbeRTT related Controls");
 838         SYSCTL_ADD_U16(&rack_sysctl_ctx,
 839             SYSCTL_CHILDREN(rack_probertt),
 840             OID_AUTO, "exit_per_hpb", CTLFLAG_RW,
 841             &rack_atexit_prtt_hbp, 130,
 842             "What percentage above goodput do we clamp CA/SS to at exit on high-BDP path 110%");
 843         SYSCTL_ADD_U16(&rack_sysctl_ctx,
 844             SYSCTL_CHILDREN(rack_probertt),
 845             OID_AUTO, "exit_per_nonhpb", CTLFLAG_RW,
 846             &rack_atexit_prtt, 130,
 847             "What percentage above goodput do we clamp CA/SS to at exit on a non high-BDP path 100%");
 848         SYSCTL_ADD_U16(&rack_sysctl_ctx,
 849             SYSCTL_CHILDREN(rack_probertt),
 850             OID_AUTO, "gp_per_mul", CTLFLAG_RW,
 851             &rack_per_of_gp_probertt, 60,
 852             "What percentage of goodput do we pace at in probertt");
 853         SYSCTL_ADD_U16(&rack_sysctl_ctx,
 854             SYSCTL_CHILDREN(rack_probertt),
 855             OID_AUTO, "gp_per_reduce", CTLFLAG_RW,
 856             &rack_per_of_gp_probertt_reduce, 10,
 857             "What percentage of goodput do we reduce every gp_srtt");
 858         SYSCTL_ADD_U16(&rack_sysctl_ctx,
 859             SYSCTL_CHILDREN(rack_probertt),
 860             OID_AUTO, "gp_per_low", CTLFLAG_RW,
 861             &rack_per_of_gp_lowthresh, 40,
 862             "What percentage of goodput do we allow the multiplier to fall to");
 863         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 864             SYSCTL_CHILDREN(rack_probertt),
 865             OID_AUTO, "time_between", CTLFLAG_RW,
 866             & rack_time_between_probertt, 96000000,
 867             "How many useconds between the lowest rtt falling must past before we enter probertt");
 868         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 869             SYSCTL_CHILDREN(rack_probertt),
 870             OID_AUTO, "safety", CTLFLAG_RW,
 871             &rack_probe_rtt_safety_val, 2000000,
 872             "If not zero, provides a maximum usecond that you can stay in probertt (2sec = 2000000)");
 873         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 874             SYSCTL_CHILDREN(rack_probertt),
 875             OID_AUTO, "sets_cwnd", CTLFLAG_RW,
 876             &rack_probe_rtt_sets_cwnd, 0,
 877             "Do we set the cwnd too (if always_lower is on)");
 878         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 879             SYSCTL_CHILDREN(rack_probertt),
 880             OID_AUTO, "maxdrainsrtts", CTLFLAG_RW,
 881             &rack_max_drain_wait, 2,
 882             "Maximum number of gp_srtt's to hold in drain waiting for flight to reach goal");
 883         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 884             SYSCTL_CHILDREN(rack_probertt),
 885             OID_AUTO, "mustdrainsrtts", CTLFLAG_RW,
 886             &rack_must_drain, 1,
 887             "We must drain this many gp_srtt's waiting for flight to reach goal");
 888         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 889             SYSCTL_CHILDREN(rack_probertt),
 890             OID_AUTO, "goal_use_min_entry", CTLFLAG_RW,
 891             &rack_probertt_use_min_rtt_entry, 1,
 892             "Should we use the min-rtt to calculate the goal rtt (else gp_srtt) at entry");
 893         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 894             SYSCTL_CHILDREN(rack_probertt),
 895             OID_AUTO, "goal_use_min_exit", CTLFLAG_RW,
 896             &rack_probertt_use_min_rtt_exit, 0,
 897             "How to set cwnd at exit, 0 - dynamic, 1 - use min-rtt, 2 - use curgprtt, 3 - entry gp-rtt");
 898         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 899             SYSCTL_CHILDREN(rack_probertt),
 900             OID_AUTO, "length_div", CTLFLAG_RW,
 901             &rack_probertt_gpsrtt_cnt_div, 0,
 902             "How many recent goodput srtt periods plus hold tim does probertt last (bottom of fraction)");
 903         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 904             SYSCTL_CHILDREN(rack_probertt),
 905             OID_AUTO, "length_mul", CTLFLAG_RW,
 906             &rack_probertt_gpsrtt_cnt_mul, 0,
 907             "How many recent goodput srtt periods plus hold tim does probertt last (top of fraction)");
 908         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 909             SYSCTL_CHILDREN(rack_probertt),
 910             OID_AUTO, "holdtim_at_target", CTLFLAG_RW,
 911             &rack_min_probertt_hold, 200000,
 912             "What is the minimum time we hold probertt at target");
 913         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 914             SYSCTL_CHILDREN(rack_probertt),
 915             OID_AUTO, "filter_life", CTLFLAG_RW,
 916             &rack_probertt_filter_life, 10000000,
 917             "What is the time for the filters life in useconds");
 918         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 919             SYSCTL_CHILDREN(rack_probertt),
 920             OID_AUTO, "lower_within", CTLFLAG_RW,
 921             &rack_probertt_lower_within, 10,
 922             "If the rtt goes lower within this percentage of the time, go into probe-rtt");
 923         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 924             SYSCTL_CHILDREN(rack_probertt),
 925             OID_AUTO, "must_move", CTLFLAG_RW,
 926             &rack_min_rtt_movement, 250,
 927             "How much is the minimum movement in rtt to count as a drop for probertt purposes");
 928         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 929             SYSCTL_CHILDREN(rack_probertt),
 930             OID_AUTO, "clear_is_cnts", CTLFLAG_RW,
 931             &rack_probertt_clear_is, 1,
 932             "Do we clear I/S counts on exiting probe-rtt");
 933         SYSCTL_ADD_S32(&rack_sysctl_ctx,
 934             SYSCTL_CHILDREN(rack_probertt),
 935             OID_AUTO, "hbp_extra_drain", CTLFLAG_RW,
 936             &rack_max_drain_hbp, 1,
 937             "How many extra drain gpsrtt's do we get in highly buffered paths");
 938         SYSCTL_ADD_S32(&rack_sysctl_ctx,
 939             SYSCTL_CHILDREN(rack_probertt),
 940             OID_AUTO, "hbp_threshold", CTLFLAG_RW,
 941             &rack_hbp_thresh, 3,
 942             "We are highly buffered if min_rtt_seen / max_rtt_seen > this-threshold");
 943
 944         rack_tracepoint = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
 945             SYSCTL_CHILDREN(rack_sysctl_root),
 946             OID_AUTO,
 947             "tp",
 948             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 949             "Rack tracepoint facility");
 950         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 951             SYSCTL_CHILDREN(rack_tracepoint),
 952             OID_AUTO, "number", CTLFLAG_RW,
 953             &rack_trace_point_config, 0,
 954             "What is the trace point number to activate (0=none, 0xffffffff = all)?");
 955         SYSCTL_ADD_U32(&rack_sysctl_ctx,
 956             SYSCTL_CHILDREN(rack_tracepoint),
 957             OID_AUTO, "bbmode", CTLFLAG_RW,
 958             &rack_trace_point_bb_mode, 4,
 959             "What is BB logging mode that is activated?");
 960         SYSCTL_ADD_S32(&rack_sysctl_ctx,
 961             SYSCTL_CHILDREN(rack_tracepoint),
 962             OID_AUTO, "count", CTLFLAG_RW,
 963             &rack_trace_point_count, 0,
 964             "How many connections will have BB logging turned on that hit the tracepoint?");
 965         /* Pacing related sysctls */
 966         rack_pacing = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
 967             SYSCTL_CHILDREN(rack_sysctl_root),
 968             OID_AUTO,
 969             "pacing",
 970             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
 971             "Pacing related Controls");
 972         SYSCTL_ADD_S32(&rack_sysctl_ctx,
 973             SYSCTL_CHILDREN(rack_pacing),
 974             OID_AUTO, "max_pace_over", CTLFLAG_RW,
 975             &rack_max_per_above, 30,
 976             "What is the maximum allowable percentage that we can pace above (so 30 = 130% of our goal)");
 977         SYSCTL_ADD_S32(&rack_sysctl_ctx,
 978             SYSCTL_CHILDREN(rack_pacing),
 979             OID_AUTO, "pace_to_one", CTLFLAG_RW,
 980             &rack_pace_one_seg, 0,
 981             "Do we allow low b/w pacing of 1MSS instead of two");
 982         SYSCTL_ADD_S32(&rack_sysctl_ctx,
 983             SYSCTL_CHILDREN(rack_pacing),
 984             OID_AUTO, "limit_wsrtt", CTLFLAG_RW,
 985             &rack_limit_time_with_srtt, 0,
 986             "Do we limit pacing time based on srtt");
 987         SYSCTL_ADD_S32(&rack_sysctl_ctx,
 988             SYSCTL_CHILDREN(rack_pacing),
 989             OID_AUTO, "init_win", CTLFLAG_RW,
 990             &rack_default_init_window, 0,
 991             "Do we have a rack initial window 0 = system default");
 992         SYSCTL_ADD_U16(&rack_sysctl_ctx,
 993             SYSCTL_CHILDREN(rack_pacing),
 994             OID_AUTO, "gp_per_ss", CTLFLAG_RW,
 995             &rack_per_of_gp_ss, 250,
 996             "If non zero, what percentage of goodput to pace at in slow start");
 997         SYSCTL_ADD_U16(&rack_sysctl_ctx,
 998             SYSCTL_CHILDREN(rack_pacing),
 999             OID_AUTO, "gp_per_ca", CTLFLAG_RW,
1000             &rack_per_of_gp_ca, 150,
1001             "If non zero, what percentage of goodput to pace at in congestion avoidance");
1002         SYSCTL_ADD_U16(&rack_sysctl_ctx,
1003             SYSCTL_CHILDREN(rack_pacing),
1004             OID_AUTO, "gp_per_rec", CTLFLAG_RW,
1005             &rack_per_of_gp_rec, 200,
1006             "If non zero, what percentage of goodput to pace at in recovery");
1007         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1008             SYSCTL_CHILDREN(rack_pacing),
1009             OID_AUTO, "pace_max_seg", CTLFLAG_RW,
1010             &rack_hptsi_segments, 40,
1011             "What size is the max for TSO segments in pacing and burst mitigation");
1012         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1013             SYSCTL_CHILDREN(rack_pacing),
1014             OID_AUTO, "burst_reduces", CTLFLAG_RW,
1015             &rack_slot_reduction, 4,
1016             "When doing only burst mitigation what is the reduce divisor");
1017         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1018             SYSCTL_CHILDREN(rack_sysctl_root),
1019             OID_AUTO, "use_pacing", CTLFLAG_RW,
1020             &rack_pace_every_seg, 0,
1021             "If set we use pacing, if clear we use only the original burst mitigation");
1022         SYSCTL_ADD_U64(&rack_sysctl_ctx,
1023             SYSCTL_CHILDREN(rack_pacing),
1024             OID_AUTO, "rate_cap", CTLFLAG_RW,
1025             &rack_bw_rate_cap, 0,
1026             "If set we apply this value to the absolute rate cap used by pacing");
1027         SYSCTL_ADD_U8(&rack_sysctl_ctx,
1028             SYSCTL_CHILDREN(rack_sysctl_root),
1029             OID_AUTO, "req_measure_cnt", CTLFLAG_RW,
1030             &rack_req_measurements, 1,
1031             "If doing dynamic pacing, how many measurements must be in before we start pacing?");
1032         /* Hardware pacing */
1033         rack_hw_pacing = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1034             SYSCTL_CHILDREN(rack_sysctl_root),
1035             OID_AUTO,
1036             "hdwr_pacing",
1037             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1038             "Pacing related Controls");
1039         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1040             SYSCTL_CHILDREN(rack_hw_pacing),
1041             OID_AUTO, "rwnd_factor", CTLFLAG_RW,
1042             &rack_hw_rwnd_factor, 2,
1043             "How many times does snd_wnd need to be bigger than pace_max_seg so we will hold off and get more acks?");
1044         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1045             SYSCTL_CHILDREN(rack_hw_pacing),
1046             OID_AUTO, "pace_enobuf_mult", CTLFLAG_RW,
1047             &rack_enobuf_hw_boost_mult, 2,
1048             "By how many time_betweens should we boost the pacing time if we see a ENOBUFS?");
1049         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1050             SYSCTL_CHILDREN(rack_hw_pacing),
1051             OID_AUTO, "pace_enobuf_max", CTLFLAG_RW,
1052             &rack_enobuf_hw_max, 2,
1053             "What is the max boost the pacing time if we see a ENOBUFS?");
1054         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1055             SYSCTL_CHILDREN(rack_hw_pacing),
1056             OID_AUTO, "pace_enobuf_min", CTLFLAG_RW,
1057             &rack_enobuf_hw_min, 2,
1058             "What is the min boost the pacing time if we see a ENOBUFS?");
1059         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1060             SYSCTL_CHILDREN(rack_hw_pacing),
1061             OID_AUTO, "enable", CTLFLAG_RW,
1062             &rack_enable_hw_pacing, 0,
1063             "Should RACK attempt to use hw pacing?");
1064         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1065             SYSCTL_CHILDREN(rack_hw_pacing),
1066             OID_AUTO, "rate_cap", CTLFLAG_RW,
1067             &rack_hw_rate_caps, 1,
1068             "Does the highest hardware pacing rate cap the rate we will send at??");
1069         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1070             SYSCTL_CHILDREN(rack_hw_pacing),
1071             OID_AUTO, "rate_min", CTLFLAG_RW,
1072             &rack_hw_rate_min, 0,
1073             "Do we need a minimum estimate of this many bytes per second in order to engage hw pacing?");
1074         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1075             SYSCTL_CHILDREN(rack_hw_pacing),
1076             OID_AUTO, "rate_to_low", CTLFLAG_RW,
1077             &rack_hw_rate_to_low, 0,
1078             "If we fall below this rate, dis-engage hw pacing?");
1079         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1080             SYSCTL_CHILDREN(rack_hw_pacing),
1081             OID_AUTO, "up_only", CTLFLAG_RW,
1082             &rack_hw_up_only, 1,
1083             "Do we allow hw pacing to lower the rate selected?");
1084         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1085             SYSCTL_CHILDREN(rack_hw_pacing),
1086             OID_AUTO, "extra_mss_precise", CTLFLAG_RW,
1087             &rack_hw_pace_extra_slots, 2,
1088             "If the rates between software and hardware match precisely how many extra time_betweens do we get?");
1089         rack_timely = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1090             SYSCTL_CHILDREN(rack_sysctl_root),
1091             OID_AUTO,
1092             "timely",
1093             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1094             "Rack Timely RTT Controls");
1095         /* Timely based GP dynmics */
1096         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1097             SYSCTL_CHILDREN(rack_timely),
1098             OID_AUTO, "upper", CTLFLAG_RW,
1099             &rack_gp_per_bw_mul_up, 2,
1100             "Rack timely upper range for equal b/w (in percentage)");
1101         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1102             SYSCTL_CHILDREN(rack_timely),
1103             OID_AUTO, "lower", CTLFLAG_RW,
1104             &rack_gp_per_bw_mul_down, 4,
1105             "Rack timely lower range for equal b/w (in percentage)");
1106         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1107             SYSCTL_CHILDREN(rack_timely),
1108             OID_AUTO, "rtt_max_mul", CTLFLAG_RW,
1109             &rack_gp_rtt_maxmul, 3,
1110             "Rack timely multiplier of lowest rtt for rtt_max");
1111         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1112             SYSCTL_CHILDREN(rack_timely),
1113             OID_AUTO, "rtt_min_div", CTLFLAG_RW,
1114             &rack_gp_rtt_mindiv, 4,
1115             "Rack timely divisor used for rtt + (rtt * mul/divisor) for check for lower rtt");
1116         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1117             SYSCTL_CHILDREN(rack_timely),
1118             OID_AUTO, "rtt_min_mul", CTLFLAG_RW,
1119             &rack_gp_rtt_minmul, 1,
1120             "Rack timely multiplier used for rtt + (rtt * mul/divisor) for check for lower rtt");
1121         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1122             SYSCTL_CHILDREN(rack_timely),
1123             OID_AUTO, "decrease", CTLFLAG_RW,
1124             &rack_gp_decrease_per, 20,
1125             "Rack timely decrease percentage of our GP multiplication factor");
1126         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1127             SYSCTL_CHILDREN(rack_timely),
1128             OID_AUTO, "increase", CTLFLAG_RW,
1129             &rack_gp_increase_per, 2,
1130             "Rack timely increase perentage of our GP multiplication factor");
1131         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1132             SYSCTL_CHILDREN(rack_timely),
1133             OID_AUTO, "lowerbound", CTLFLAG_RW,
1134             &rack_per_lower_bound, 50,
1135             "Rack timely lowest percentage we allow GP multiplier to fall to");
1136         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1137             SYSCTL_CHILDREN(rack_timely),
1138             OID_AUTO, "upperboundss", CTLFLAG_RW,
1139             &rack_per_upper_bound_ss, 0,
1140             "Rack timely highest percentage we allow GP multiplier in SS to raise to (0 is no upperbound)");
1141         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1142             SYSCTL_CHILDREN(rack_timely),
1143             OID_AUTO, "upperboundca", CTLFLAG_RW,
1144             &rack_per_upper_bound_ca, 0,
1145             "Rack timely highest percentage we allow GP multiplier to CA raise to (0 is no upperbound)");
1146         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1147             SYSCTL_CHILDREN(rack_timely),
1148             OID_AUTO, "dynamicgp", CTLFLAG_RW,
1149             &rack_do_dyn_mul, 0,
1150             "Rack timely do we enable dynmaic timely goodput by default");
1151         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1152             SYSCTL_CHILDREN(rack_timely),
1153             OID_AUTO, "no_rec_red", CTLFLAG_RW,
1154             &rack_gp_no_rec_chg, 1,
1155             "Rack timely do we prohibit the recovery multiplier from being lowered");
1156         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1157             SYSCTL_CHILDREN(rack_timely),
1158             OID_AUTO, "red_clear_cnt", CTLFLAG_RW,
1159             &rack_timely_dec_clear, 6,
1160             "Rack timely what threshold do we count to before another boost during b/w decent");
1161         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1162             SYSCTL_CHILDREN(rack_timely),
1163             OID_AUTO, "max_push_rise", CTLFLAG_RW,
1164             &rack_timely_max_push_rise, 3,
1165             "Rack timely how many times do we push up with b/w increase");
1166         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1167             SYSCTL_CHILDREN(rack_timely),
1168             OID_AUTO, "max_push_drop", CTLFLAG_RW,
1169             &rack_timely_max_push_drop, 3,
1170             "Rack timely how many times do we push back on b/w decent");
1171         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1172             SYSCTL_CHILDREN(rack_timely),
1173             OID_AUTO, "min_segs", CTLFLAG_RW,
1174             &rack_timely_min_segs, 4,
1175             "Rack timely when setting the cwnd what is the min num segments");
1176         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1177             SYSCTL_CHILDREN(rack_timely),
1178             OID_AUTO, "noback_max", CTLFLAG_RW,
1179             &rack_use_max_for_nobackoff, 0,
1180             "Rack timely when deciding if to backoff on a loss, do we use under max rtt else min");
1181         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1182             SYSCTL_CHILDREN(rack_timely),
1183             OID_AUTO, "interim_timely_only", CTLFLAG_RW,
1184             &rack_timely_int_timely_only, 0,
1185             "Rack timely when doing interim timely's do we only do timely (no b/w consideration)");
1186         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1187             SYSCTL_CHILDREN(rack_timely),
1188             OID_AUTO, "nonstop", CTLFLAG_RW,
1189             &rack_timely_no_stopping, 0,
1190             "Rack timely don't stop increase");
1191         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1192             SYSCTL_CHILDREN(rack_timely),
1193             OID_AUTO, "dec_raise_thresh", CTLFLAG_RW,
1194             &rack_down_raise_thresh, 100,
1195             "If the CA or SS is below this threshold raise on the first 3 b/w lowers (0=always)");
1196         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1197             SYSCTL_CHILDREN(rack_timely),
1198             OID_AUTO, "bottom_drag_segs", CTLFLAG_RW,
1199             &rack_req_segs, 1,
1200             "Bottom dragging if not these many segments outstanding and room");
1201
1202         /* TLP and Rack related parameters */
1203         rack_tlp = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1204             SYSCTL_CHILDREN(rack_sysctl_root),
1205             OID_AUTO,
1206             "tlp",
1207             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1208             "TLP and Rack related Controls");
1209         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1210             SYSCTL_CHILDREN(rack_tlp),
1211             OID_AUTO, "use_rrr", CTLFLAG_RW,
1212             &use_rack_rr, 1,
1213             "Do we use Rack Rapid Recovery");
1214         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1215             SYSCTL_CHILDREN(rack_tlp),
1216             OID_AUTO, "post_rec_labc", CTLFLAG_RW,
1217             &rack_max_abc_post_recovery, 2,
1218             "Since we do early recovery, do we override the l_abc to a value, if so what?");
1219         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1220             SYSCTL_CHILDREN(rack_tlp),
1221             OID_AUTO, "nonrxt_use_cr", CTLFLAG_RW,
1222             &rack_non_rxt_use_cr, 0,
1223             "Do we use ss/ca rate if in recovery we are transmitting a new data chunk");
1224         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1225             SYSCTL_CHILDREN(rack_tlp),
1226             OID_AUTO, "tlpmethod", CTLFLAG_RW,
1227             &rack_tlp_threshold_use, TLP_USE_TWO_ONE,
1228             "What method do we do for TLP time calc 0=no-de-ack-comp, 1=ID, 2=2.1, 3=2.2");
1229         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1230             SYSCTL_CHILDREN(rack_tlp),
1231             OID_AUTO, "limit", CTLFLAG_RW,
1232             &rack_tlp_limit, 2,
1233             "How many TLP's can be sent without sending new data");
1234         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1235             SYSCTL_CHILDREN(rack_tlp),
1236             OID_AUTO, "use_greater", CTLFLAG_RW,
1237             &rack_tlp_use_greater, 1,
1238             "Should we use the rack_rtt time if its greater than srtt");
1239         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1240             SYSCTL_CHILDREN(rack_tlp),
1241             OID_AUTO, "tlpminto", CTLFLAG_RW,
1242             &rack_tlp_min, 10000,
1243             "TLP minimum timeout per the specification (in microseconds)");
1244         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1245             SYSCTL_CHILDREN(rack_tlp),
1246             OID_AUTO, "send_oldest", CTLFLAG_RW,
1247             &rack_always_send_oldest, 0,
1248             "Should we always send the oldest TLP and RACK-TLP");
1249         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1250             SYSCTL_CHILDREN(rack_tlp),
1251             OID_AUTO, "rack_tlimit", CTLFLAG_RW,
1252             &rack_limited_retran, 0,
1253             "How many times can a rack timeout drive out sends");
1254         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1255             SYSCTL_CHILDREN(rack_tlp),
1256             OID_AUTO, "tlp_cwnd_flag", CTLFLAG_RW,
1257             &rack_lower_cwnd_at_tlp, 0,
1258             "When a TLP completes a retran should we enter recovery");
1259         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1260             SYSCTL_CHILDREN(rack_tlp),
1261             OID_AUTO, "reorder_thresh", CTLFLAG_RW,
1262             &rack_reorder_thresh, 2,
1263             "What factor for rack will be added when seeing reordering (shift right)");
1264         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1265             SYSCTL_CHILDREN(rack_tlp),
1266             OID_AUTO, "rtt_tlp_thresh", CTLFLAG_RW,
1267             &rack_tlp_thresh, 1,
1268             "What divisor for TLP rtt/retran will be added (1=rtt, 2=1/2 rtt etc)");
1269         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1270             SYSCTL_CHILDREN(rack_tlp),
1271             OID_AUTO, "reorder_fade", CTLFLAG_RW,
1272             &rack_reorder_fade, 60000000,
1273             "Does reorder detection fade, if so how many microseconds (0 means never)");
1274         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1275             SYSCTL_CHILDREN(rack_tlp),
1276             OID_AUTO, "pktdelay", CTLFLAG_RW,
1277             &rack_pkt_delay, 1000,
1278             "Extra RACK time (in microseconds) besides reordering thresh");
1279
1280         /* Timer related controls */
1281         rack_timers = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1282             SYSCTL_CHILDREN(rack_sysctl_root),
1283             OID_AUTO,
1284             "timers",
1285             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1286             "Timer related controls");
1287         SYSCTL_ADD_U32(&rack_sysctl_ctx,
1288             SYSCTL_CHILDREN(rack_timers),
1289             OID_AUTO, "persmin", CTLFLAG_RW,
1290             &rack_persist_min, 250000,
1291             "What is the minimum time in microseconds between persists");
1292         SYSCTL_ADD_U32(&rack_sysctl_ctx,
1293             SYSCTL_CHILDREN(rack_timers),
1294             OID_AUTO, "persmax", CTLFLAG_RW,
1295             &rack_persist_max, 2000000,
1296             "What is the largest delay in microseconds between persists");
1297         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1298             SYSCTL_CHILDREN(rack_timers),
1299             OID_AUTO, "delayed_ack", CTLFLAG_RW,
1300             &rack_delayed_ack_time, 40000,
1301             "Delayed ack time (40ms in microseconds)");
1302         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1303             SYSCTL_CHILDREN(rack_timers),
1304             OID_AUTO, "minrto", CTLFLAG_RW,
1305             &rack_rto_min, 30000,
1306             "Minimum RTO in microseconds -- set with caution below 1000 due to TLP");
1307         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1308             SYSCTL_CHILDREN(rack_timers),
1309             OID_AUTO, "maxrto", CTLFLAG_RW,
1310             &rack_rto_max, 4000000,
1311             "Maximum RTO in microseconds -- should be at least as large as min_rto");
1312         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1313             SYSCTL_CHILDREN(rack_timers),
1314             OID_AUTO, "minto", CTLFLAG_RW,
1315             &rack_min_to, 1000,
1316             "Minimum rack timeout in microseconds");
1317         /* Measure controls */
1318         rack_measure = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1319             SYSCTL_CHILDREN(rack_sysctl_root),
1320             OID_AUTO,
1321             "measure",
1322             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1323             "Measure related controls");
1324         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1325             SYSCTL_CHILDREN(rack_measure),
1326             OID_AUTO, "wma_divisor", CTLFLAG_RW,
1327             &rack_wma_divisor, 8,
1328             "When doing b/w calculation what is the  divisor for the WMA");
1329         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1330             SYSCTL_CHILDREN(rack_measure),
1331             OID_AUTO, "end_cwnd", CTLFLAG_RW,
1332             &rack_cwnd_block_ends_measure, 0,
1333             "Does a cwnd just-return end the measurement window (app limited)");
1334         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1335             SYSCTL_CHILDREN(rack_measure),
1336             OID_AUTO, "end_rwnd", CTLFLAG_RW,
1337             &rack_rwnd_block_ends_measure, 0,
1338             "Does an rwnd just-return end the measurement window (app limited -- not persists)");
1339         SYSCTL_ADD_U32(&rack_sysctl_ctx,
1340             SYSCTL_CHILDREN(rack_measure),
1341             OID_AUTO, "min_target", CTLFLAG_RW,
1342             &rack_def_data_window, 20,
1343             "What is the minimum target window (in mss) for a GP measurements");
1344         SYSCTL_ADD_U32(&rack_sysctl_ctx,
1345             SYSCTL_CHILDREN(rack_measure),
1346             OID_AUTO, "goal_bdp", CTLFLAG_RW,
1347             &rack_goal_bdp, 2,
1348             "What is the goal BDP to measure");
1349         SYSCTL_ADD_U32(&rack_sysctl_ctx,
1350             SYSCTL_CHILDREN(rack_measure),
1351             OID_AUTO, "min_srtts", CTLFLAG_RW,
1352             &rack_min_srtts, 1,
1353             "What is the goal BDP to measure");
1354         SYSCTL_ADD_U32(&rack_sysctl_ctx,
1355             SYSCTL_CHILDREN(rack_measure),
1356             OID_AUTO, "min_measure_tim", CTLFLAG_RW,
1357             &rack_min_measure_usec, 0,
1358             "What is the Minimum time time for a measurement if 0, this is off");
1359         /* Features */
1360         rack_features = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1361             SYSCTL_CHILDREN(rack_sysctl_root),
1362             OID_AUTO,
1363             "features",
1364             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1365             "Feature controls");
1366         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1367             SYSCTL_CHILDREN(rack_features),
1368             OID_AUTO, "cmpack", CTLFLAG_RW,
1369             &rack_use_cmp_acks, 1,
1370             "Should RACK have LRO send compressed acks");
1371         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1372             SYSCTL_CHILDREN(rack_features),
1373             OID_AUTO, "fsb", CTLFLAG_RW,
1374             &rack_use_fsb, 1,
1375             "Should RACK use the fast send block?");
1376         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1377             SYSCTL_CHILDREN(rack_features),
1378             OID_AUTO, "rfo", CTLFLAG_RW,
1379             &rack_use_rfo, 1,
1380             "Should RACK use rack_fast_output()?");
1381         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1382             SYSCTL_CHILDREN(rack_features),
1383             OID_AUTO, "rsmrfo", CTLFLAG_RW,
1384             &rack_use_rsm_rfo, 1,
1385             "Should RACK use rack_fast_rsm_output()?");
1386         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1387             SYSCTL_CHILDREN(rack_features),
1388             OID_AUTO, "non_paced_lro_queue", CTLFLAG_RW,
1389             &rack_enable_mqueue_for_nonpaced, 0,
1390             "Should RACK use mbuf queuing for non-paced connections");
1391         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1392             SYSCTL_CHILDREN(rack_features),
1393             OID_AUTO, "hystartplusplus", CTLFLAG_RW,
1394             &rack_do_hystart, 0,
1395             "Should RACK enable HyStart++ on connections?");
1396         /* Misc rack controls */
1397         rack_misc = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
1398             SYSCTL_CHILDREN(rack_sysctl_root),
1399             OID_AUTO,
1400             "misc",
1401             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1402             "Misc related controls");
1403 #ifdef TCP_ACCOUNTING
1404         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1405             SYSCTL_CHILDREN(rack_misc),
1406             OID_AUTO, "tcp_acct", CTLFLAG_RW,
1407             &rack_tcp_accounting, 0,
1408             "Should we turn on TCP accounting for all rack sessions?");
1409 #endif
1410         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1411             SYSCTL_CHILDREN(rack_misc),
1412             OID_AUTO, "apply_rtt_with_low_conf", CTLFLAG_RW,
1413             &rack_apply_rtt_with_reduced_conf, 0,
1414             "When a persist or keep-alive probe is not answered do we calculate rtt on subsequent answers?");
1415         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1416             SYSCTL_CHILDREN(rack_misc),
1417             OID_AUTO, "rack_dsack_ctl", CTLFLAG_RW,
1418             &rack_dsack_std_based, 3,
1419             "How do we process dsack with respect to rack timers, bit field, 3 is standards based?");
1420         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1421             SYSCTL_CHILDREN(rack_misc),
1422             OID_AUTO, "prr_addback_max", CTLFLAG_RW,
1423             &rack_prr_addbackmax, 2,
1424             "What is the maximum number of MSS we allow to be added back if prr can't send all its data?");
1425         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1426             SYSCTL_CHILDREN(rack_misc),
1427             OID_AUTO, "stats_gets_ms", CTLFLAG_RW,
1428             &rack_stats_gets_ms_rtt, 1,
1429             "What do we feed the stats framework (1 = ms_rtt, 0 = us_rtt, 2 = ms_rtt from hdwr, > 2 usec rtt from hdwr)?");
1430         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1431             SYSCTL_CHILDREN(rack_misc),
1432             OID_AUTO, "clientlowbuf", CTLFLAG_RW,
1433             &rack_client_low_buf, 0,
1434             "Client low buffer level (below this we are more aggressive in DGP exiting recovery (0 = off)?");
1435         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1436             SYSCTL_CHILDREN(rack_misc),
1437             OID_AUTO, "defprofile", CTLFLAG_RW,
1438             &rack_def_profile, 0,
1439             "Should RACK use a default profile (0=no, num == profile num)?");
1440         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1441             SYSCTL_CHILDREN(rack_misc),
1442             OID_AUTO, "shared_cwnd", CTLFLAG_RW,
1443             &rack_enable_shared_cwnd, 1,
1444             "Should RACK try to use the shared cwnd on connections where allowed");
1445         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1446             SYSCTL_CHILDREN(rack_misc),
1447             OID_AUTO, "limits_on_scwnd", CTLFLAG_RW,
1448             &rack_limits_scwnd, 1,
1449             "Should RACK place low end time limits on the shared cwnd feature");
1450         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1451             SYSCTL_CHILDREN(rack_misc),
1452             OID_AUTO, "iMac_dack", CTLFLAG_RW,
1453             &rack_use_imac_dack, 0,
1454             "Should RACK try to emulate iMac delayed ack");
1455         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1456             SYSCTL_CHILDREN(rack_misc),
1457             OID_AUTO, "no_prr", CTLFLAG_RW,
1458             &rack_disable_prr, 0,
1459             "Should RACK not use prr and only pace (must have pacing on)");
1460         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1461             SYSCTL_CHILDREN(rack_misc),
1462             OID_AUTO, "bb_verbose", CTLFLAG_RW,
1463             &rack_verbose_logging, 0,
1464             "Should RACK black box logging be verbose");
1465         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1466             SYSCTL_CHILDREN(rack_misc),
1467             OID_AUTO, "data_after_close", CTLFLAG_RW,
1468             &rack_ignore_data_after_close, 1,
1469             "Do we hold off sending a RST until all pending data is ack'd");
1470         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1471             SYSCTL_CHILDREN(rack_misc),
1472             OID_AUTO, "no_sack_needed", CTLFLAG_RW,
1473             &rack_sack_not_required, 1,
1474             "Do we allow rack to run on connections not supporting SACK");
1475         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1476             SYSCTL_CHILDREN(rack_misc),
1477             OID_AUTO, "prr_sendalot", CTLFLAG_RW,
1478             &rack_send_a_lot_in_prr, 1,
1479             "Send a lot in prr");
1480         SYSCTL_ADD_S32(&rack_sysctl_ctx,
1481             SYSCTL_CHILDREN(rack_misc),
1482             OID_AUTO, "autoscale", CTLFLAG_RW,
1483             &rack_autosndbuf_inc, 20,
1484             "What percentage should rack scale up its snd buffer by?");
1485         /* Sack Attacker detection stuff */
1486         SYSCTL_ADD_U32(&rack_sysctl_ctx,
1487             SYSCTL_CHILDREN(rack_attack),
1488             OID_AUTO, "detect_highsackratio", CTLFLAG_RW,
1489             &rack_highest_sack_thresh_seen, 0,
1490             "Highest sack to ack ratio seen");
1491         SYSCTL_ADD_U32(&rack_sysctl_ctx,
1492             SYSCTL_CHILDREN(rack_attack),
1493             OID_AUTO, "detect_highmoveratio", CTLFLAG_RW,
1494             &rack_highest_move_thresh_seen, 0,
1495             "Highest move to non-move ratio seen");
1496         rack_ack_total = counter_u64_alloc(M_WAITOK);
1497         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1498             SYSCTL_CHILDREN(rack_attack),
1499             OID_AUTO, "acktotal", CTLFLAG_RD,
1500             &rack_ack_total,
1501             "Total number of Ack's");
1502         rack_express_sack = counter_u64_alloc(M_WAITOK);
1503         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1504             SYSCTL_CHILDREN(rack_attack),
1505             OID_AUTO, "exp_sacktotal", CTLFLAG_RD,
1506             &rack_express_sack,
1507             "Total expresss number of Sack's");
1508         rack_sack_total = counter_u64_alloc(M_WAITOK);
1509         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1510             SYSCTL_CHILDREN(rack_attack),
1511             OID_AUTO, "sacktotal", CTLFLAG_RD,
1512             &rack_sack_total,
1513             "Total number of SACKs");
1514         rack_move_none = counter_u64_alloc(M_WAITOK);
1515         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1516             SYSCTL_CHILDREN(rack_attack),
1517             OID_AUTO, "move_none", CTLFLAG_RD,
1518             &rack_move_none,
1519             "Total number of SACK index reuse of positions under threshold");
1520         rack_move_some = counter_u64_alloc(M_WAITOK);
1521         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1522             SYSCTL_CHILDREN(rack_attack),
1523             OID_AUTO, "move_some", CTLFLAG_RD,
1524             &rack_move_some,
1525             "Total number of SACK index reuse of positions over threshold");
1526         rack_sack_attacks_detected = counter_u64_alloc(M_WAITOK);
1527         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1528             SYSCTL_CHILDREN(rack_attack),
1529             OID_AUTO, "attacks", CTLFLAG_RD,
1530             &rack_sack_attacks_detected,
1531             "Total number of SACK attackers that had sack disabled");
1532         rack_sack_attacks_reversed = counter_u64_alloc(M_WAITOK);
1533         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1534             SYSCTL_CHILDREN(rack_attack),
1535             OID_AUTO, "reversed", CTLFLAG_RD,
1536             &rack_sack_attacks_reversed,
1537             "Total number of SACK attackers that were later determined false positive");
1538         rack_sack_used_next_merge = counter_u64_alloc(M_WAITOK);
1539         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1540             SYSCTL_CHILDREN(rack_attack),
1541             OID_AUTO, "nextmerge", CTLFLAG_RD,
1542             &rack_sack_used_next_merge,
1543             "Total number of times we used the next merge");
1544         rack_sack_used_prev_merge = counter_u64_alloc(M_WAITOK);
1545         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1546             SYSCTL_CHILDREN(rack_attack),
1547             OID_AUTO, "prevmerge", CTLFLAG_RD,
1548             &rack_sack_used_prev_merge,
1549             "Total number of times we used the prev merge");
1550         /* Counters */
1551         rack_fto_send = counter_u64_alloc(M_WAITOK);
1552         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1553             SYSCTL_CHILDREN(rack_counters),
1554             OID_AUTO, "fto_send", CTLFLAG_RD,
1555             &rack_fto_send, "Total number of rack_fast_output sends");
1556         rack_fto_rsm_send = counter_u64_alloc(M_WAITOK);
1557         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1558             SYSCTL_CHILDREN(rack_counters),
1559             OID_AUTO, "fto_rsm_send", CTLFLAG_RD,
1560             &rack_fto_rsm_send, "Total number of rack_fast_rsm_output sends");
1561         rack_nfto_resend = counter_u64_alloc(M_WAITOK);
1562         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1563             SYSCTL_CHILDREN(rack_counters),
1564             OID_AUTO, "nfto_resend", CTLFLAG_RD,
1565             &rack_nfto_resend, "Total number of rack_output retransmissions");
1566         rack_non_fto_send = counter_u64_alloc(M_WAITOK);
1567         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1568             SYSCTL_CHILDREN(rack_counters),
1569             OID_AUTO, "nfto_send", CTLFLAG_RD,
1570             &rack_non_fto_send, "Total number of rack_output first sends");
1571         rack_extended_rfo = counter_u64_alloc(M_WAITOK);
1572         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1573             SYSCTL_CHILDREN(rack_counters),
1574             OID_AUTO, "rfo_extended", CTLFLAG_RD,
1575             &rack_extended_rfo, "Total number of times we extended rfo");
1576
1577         rack_hw_pace_init_fail = counter_u64_alloc(M_WAITOK);
1578         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1579             SYSCTL_CHILDREN(rack_counters),
1580             OID_AUTO, "hwpace_init_fail", CTLFLAG_RD,
1581             &rack_hw_pace_init_fail, "Total number of times we failed to initialize hw pacing");
1582         rack_hw_pace_lost = counter_u64_alloc(M_WAITOK);
1583
1584         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1585             SYSCTL_CHILDREN(rack_counters),
1586             OID_AUTO, "hwpace_lost", CTLFLAG_RD,
1587             &rack_hw_pace_lost, "Total number of times we failed to initialize hw pacing");
1588         rack_tlp_tot = counter_u64_alloc(M_WAITOK);
1589         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1590             SYSCTL_CHILDREN(rack_counters),
1591             OID_AUTO, "tlp_to_total", CTLFLAG_RD,
1592             &rack_tlp_tot,
1593             "Total number of tail loss probe expirations");
1594         rack_tlp_newdata = counter_u64_alloc(M_WAITOK);
1595         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1596             SYSCTL_CHILDREN(rack_counters),
1597             OID_AUTO, "tlp_new", CTLFLAG_RD,
1598             &rack_tlp_newdata,
1599             "Total number of tail loss probe sending new data");
1600         rack_tlp_retran = counter_u64_alloc(M_WAITOK);
1601         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1602             SYSCTL_CHILDREN(rack_counters),
1603             OID_AUTO, "tlp_retran", CTLFLAG_RD,
1604             &rack_tlp_retran,
1605             "Total number of tail loss probe sending retransmitted data");
1606         rack_tlp_retran_bytes = counter_u64_alloc(M_WAITOK);
1607         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1608             SYSCTL_CHILDREN(rack_counters),
1609             OID_AUTO, "tlp_retran_bytes", CTLFLAG_RD,
1610             &rack_tlp_retran_bytes,
1611             "Total bytes of tail loss probe sending retransmitted data");
1612         rack_to_tot = counter_u64_alloc(M_WAITOK);
1613         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1614             SYSCTL_CHILDREN(rack_counters),
1615             OID_AUTO, "rack_to_tot", CTLFLAG_RD,
1616             &rack_to_tot,
1617             "Total number of times the rack to expired");
1618         rack_saw_enobuf = counter_u64_alloc(M_WAITOK);
1619         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1620             SYSCTL_CHILDREN(rack_counters),
1621             OID_AUTO, "saw_enobufs", CTLFLAG_RD,
1622             &rack_saw_enobuf,
1623             "Total number of times a sends returned enobuf for non-hdwr paced connections");
1624         rack_saw_enobuf_hw = counter_u64_alloc(M_WAITOK);
1625         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1626             SYSCTL_CHILDREN(rack_counters),
1627             OID_AUTO, "saw_enobufs_hw", CTLFLAG_RD,
1628             &rack_saw_enobuf_hw,
1629             "Total number of times a send returned enobuf for hdwr paced connections");
1630         rack_saw_enetunreach = counter_u64_alloc(M_WAITOK);
1631         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1632             SYSCTL_CHILDREN(rack_counters),
1633             OID_AUTO, "saw_enetunreach", CTLFLAG_RD,
1634             &rack_saw_enetunreach,
1635             "Total number of times a send received a enetunreachable");
1636         rack_hot_alloc = counter_u64_alloc(M_WAITOK);
1637         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1638             SYSCTL_CHILDREN(rack_counters),
1639             OID_AUTO, "alloc_hot", CTLFLAG_RD,
1640             &rack_hot_alloc,
1641             "Total allocations from the top of our list");
1642         rack_to_alloc = counter_u64_alloc(M_WAITOK);
1643         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1644             SYSCTL_CHILDREN(rack_counters),
1645             OID_AUTO, "allocs", CTLFLAG_RD,
1646             &rack_to_alloc,
1647             "Total allocations of tracking structures");
1648         rack_to_alloc_hard = counter_u64_alloc(M_WAITOK);
1649         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1650             SYSCTL_CHILDREN(rack_counters),
1651             OID_AUTO, "allochard", CTLFLAG_RD,
1652             &rack_to_alloc_hard,
1653             "Total allocations done with sleeping the hard way");
1654         rack_to_alloc_emerg = counter_u64_alloc(M_WAITOK);
1655         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1656             SYSCTL_CHILDREN(rack_counters),
1657             OID_AUTO, "allocemerg", CTLFLAG_RD,
1658             &rack_to_alloc_emerg,
1659             "Total allocations done from emergency cache");
1660         rack_to_alloc_limited = counter_u64_alloc(M_WAITOK);
1661         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1662             SYSCTL_CHILDREN(rack_counters),
1663             OID_AUTO, "alloc_limited", CTLFLAG_RD,
1664             &rack_to_alloc_limited,
1665             "Total allocations dropped due to limit");
1666         rack_alloc_limited_conns = counter_u64_alloc(M_WAITOK);
1667         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1668             SYSCTL_CHILDREN(rack_counters),
1669             OID_AUTO, "alloc_limited_conns", CTLFLAG_RD,
1670             &rack_alloc_limited_conns,
1671             "Connections with allocations dropped due to limit");
1672         rack_split_limited = counter_u64_alloc(M_WAITOK);
1673         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1674             SYSCTL_CHILDREN(rack_counters),
1675             OID_AUTO, "split_limited", CTLFLAG_RD,
1676             &rack_split_limited,
1677             "Split allocations dropped due to limit");
1678         rack_persists_sends = counter_u64_alloc(M_WAITOK);
1679         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1680             SYSCTL_CHILDREN(rack_counters),
1681             OID_AUTO, "persist_sends", CTLFLAG_RD,
1682             &rack_persists_sends,
1683             "Number of times we sent a persist probe");
1684         rack_persists_acks = counter_u64_alloc(M_WAITOK);
1685         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1686             SYSCTL_CHILDREN(rack_counters),
1687             OID_AUTO, "persist_acks", CTLFLAG_RD,
1688             &rack_persists_acks,
1689             "Number of times a persist probe was acked");
1690         rack_persists_loss = counter_u64_alloc(M_WAITOK);
1691         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1692             SYSCTL_CHILDREN(rack_counters),
1693             OID_AUTO, "persist_loss", CTLFLAG_RD,
1694             &rack_persists_loss,
1695             "Number of times we detected a lost persist probe (no ack)");
1696         rack_persists_lost_ends = counter_u64_alloc(M_WAITOK);
1697         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1698             SYSCTL_CHILDREN(rack_counters),
1699             OID_AUTO, "persist_loss_ends", CTLFLAG_RD,
1700             &rack_persists_lost_ends,
1701             "Number of lost persist probe (no ack) that the run ended with a PERSIST abort");
1702 #ifdef INVARIANTS
1703         rack_adjust_map_bw = counter_u64_alloc(M_WAITOK);
1704         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1705             SYSCTL_CHILDREN(rack_counters),
1706             OID_AUTO, "map_adjust_req", CTLFLAG_RD,
1707             &rack_adjust_map_bw,
1708             "Number of times we hit the case where the sb went up and down on a sendmap entry");
1709 #endif
1710         rack_multi_single_eq = counter_u64_alloc(M_WAITOK);
1711         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1712             SYSCTL_CHILDREN(rack_counters),
1713             OID_AUTO, "cmp_ack_equiv", CTLFLAG_RD,
1714             &rack_multi_single_eq,
1715             "Number of compressed acks total represented");
1716         rack_proc_non_comp_ack = counter_u64_alloc(M_WAITOK);
1717         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1718             SYSCTL_CHILDREN(rack_counters),
1719             OID_AUTO, "cmp_ack_not", CTLFLAG_RD,
1720             &rack_proc_non_comp_ack,
1721             "Number of non compresseds acks that we processed");
1722
1723
1724         rack_sack_proc_all = counter_u64_alloc(M_WAITOK);
1725         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1726             SYSCTL_CHILDREN(rack_counters),
1727             OID_AUTO, "sack_long", CTLFLAG_RD,
1728             &rack_sack_proc_all,
1729             "Total times we had to walk whole list for sack processing");
1730         rack_sack_proc_restart = counter_u64_alloc(M_WAITOK);
1731         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1732             SYSCTL_CHILDREN(rack_counters),
1733             OID_AUTO, "sack_restart", CTLFLAG_RD,
1734             &rack_sack_proc_restart,
1735             "Total times we had to walk whole list due to a restart");
1736         rack_sack_proc_short = counter_u64_alloc(M_WAITOK);
1737         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1738             SYSCTL_CHILDREN(rack_counters),
1739             OID_AUTO, "sack_short", CTLFLAG_RD,
1740             &rack_sack_proc_short,
1741             "Total times we took shortcut for sack processing");
1742         rack_sack_skipped_acked = counter_u64_alloc(M_WAITOK);
1743         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1744             SYSCTL_CHILDREN(rack_attack),
1745             OID_AUTO, "skipacked", CTLFLAG_RD,
1746             &rack_sack_skipped_acked,
1747             "Total number of times we skipped previously sacked");
1748         rack_sack_splits = counter_u64_alloc(M_WAITOK);
1749         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1750             SYSCTL_CHILDREN(rack_attack),
1751             OID_AUTO, "ofsplit", CTLFLAG_RD,
1752             &rack_sack_splits,
1753             "Total number of times we did the old fashion tree split");
1754         rack_input_idle_reduces = counter_u64_alloc(M_WAITOK);
1755         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1756             SYSCTL_CHILDREN(rack_counters),
1757             OID_AUTO, "idle_reduce_oninput", CTLFLAG_RD,
1758             &rack_input_idle_reduces,
1759             "Total number of idle reductions on input");
1760         rack_collapsed_win = counter_u64_alloc(M_WAITOK);
1761         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1762             SYSCTL_CHILDREN(rack_counters),
1763             OID_AUTO, "collapsed_win", CTLFLAG_RD,
1764             &rack_collapsed_win,
1765             "Total number of collapsed windows");
1766         rack_try_scwnd = counter_u64_alloc(M_WAITOK);
1767         SYSCTL_ADD_COUNTER_U64(&rack_sysctl_ctx,
1768             SYSCTL_CHILDREN(rack_counters),
1769             OID_AUTO, "tried_scwnd", CTLFLAG_RD,
1770             &rack_try_scwnd,
1771             "Total number of scwnd attempts");
1772         COUNTER_ARRAY_ALLOC(rack_out_size, TCP_MSS_ACCT_SIZE, M_WAITOK);
1773         SYSCTL_ADD_COUNTER_U64_ARRAY(&rack_sysctl_ctx, SYSCTL_CHILDREN(rack_sysctl_root),
1774             OID_AUTO, "outsize", CTLFLAG_RD,
1775             rack_out_size, TCP_MSS_ACCT_SIZE, "MSS send sizes");
1776         COUNTER_ARRAY_ALLOC(rack_opts_arry, RACK_OPTS_SIZE, M_WAITOK);
1777         SYSCTL_ADD_COUNTER_U64_ARRAY(&rack_sysctl_ctx, SYSCTL_CHILDREN(rack_sysctl_root),
1778             OID_AUTO, "opts", CTLFLAG_RD,
1779             rack_opts_arry, RACK_OPTS_SIZE, "RACK Option Stats");
1780         SYSCTL_ADD_PROC(&rack_sysctl_ctx,
1781             SYSCTL_CHILDREN(rack_sysctl_root),
1782             OID_AUTO, "clear", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
1783             &rack_clear_counter, 0, sysctl_rack_clear, "IU", "Clear counters");
1784 }
1785
1786 static __inline int
1787 rb_map_cmp(struct rack_sendmap *b, struct rack_sendmap *a)
1788 {
1789         if (SEQ_GEQ(b->r_start, a->r_start) &&
1790             SEQ_LT(b->r_start, a->r_end)) {
1791                 /*
1792                  * The entry b is within the
1793                  * block a. i.e.:
1794                  * a --   |-------------|
1795                  * b --   |----|
1796                  * <or>
1797                  * b --       |------|
1798                  * <or>
1799                  * b --       |-----------|
1800                  */
1801                 return (0);
1802         } else if (SEQ_GEQ(b->r_start, a->r_end)) {
1803                 /*
1804                  * b falls as either the next
1805                  * sequence block after a so a
1806                  * is said to be smaller than b.
1807                  * i.e:
1808                  * a --   |------|
1809                  * b --          |--------|
1810                  * or
1811                  * b --              |-----|
1812                  */
1813                 return (1);
1814         }
1815         /*
1816          * Whats left is where a is
1817          * larger than b. i.e:
1818          * a --         |-------|
1819          * b --  |---|
1820          * or even possibly
1821          * b --   |--------------|
1822          */
1823         return (-1);
1824 }
1825
1826 RB_PROTOTYPE(rack_rb_tree_head, rack_sendmap, r_next, rb_map_cmp);
1827 RB_GENERATE(rack_rb_tree_head, rack_sendmap, r_next, rb_map_cmp);
1828
1829 static uint32_t
1830 rc_init_window(struct tcp_rack *rack)
1831 {
1832         uint32_t win;
1833
1834         if (rack->rc_init_win == 0) {
1835                 /*
1836                  * Nothing set by the user, use the system stack
1837                  * default.
1838                  */
1839                 return (tcp_compute_initwnd(tcp_maxseg(rack->rc_tp)));
1840         }
1841         win = ctf_fixed_maxseg(rack->rc_tp) * rack->rc_init_win;
1842         return (win);
1843 }
1844
1845 static uint64_t
1846 rack_get_fixed_pacing_bw(struct tcp_rack *rack)
1847 {
1848         if (IN_FASTRECOVERY(rack->rc_tp->t_flags))
1849                 return (rack->r_ctl.rc_fixed_pacing_rate_rec);
1850         else if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh)
1851                 return (rack->r_ctl.rc_fixed_pacing_rate_ss);
1852         else
1853                 return (rack->r_ctl.rc_fixed_pacing_rate_ca);
1854 }
1855
1856 static uint64_t
1857 rack_get_bw(struct tcp_rack *rack)
1858 {
1859         if (rack->use_fixed_rate) {
1860                 /* Return the fixed pacing rate */
1861                 return (rack_get_fixed_pacing_bw(rack));
1862         }
1863         if (rack->r_ctl.gp_bw == 0) {
1864                 /*
1865                  * We have yet no b/w measurement,
1866                  * if we have a user set initial bw
1867                  * return it. If we don't have that and
1868                  * we have an srtt, use the tcp IW (10) to
1869                  * calculate a fictional b/w over the SRTT
1870                  * which is more or less a guess. Note
1871                  * we don't use our IW from rack on purpose
1872                  * so if we have like IW=30, we are not
1873                  * calculating a "huge" b/w.
1874                  */
1875                 uint64_t bw, srtt;
1876                 if (rack->r_ctl.init_rate)
1877                         return (rack->r_ctl.init_rate);
1878
1879                 /* Has the user set a max peak rate? */
1880 #ifdef NETFLIX_PEAKRATE
1881                 if (rack->rc_tp->t_maxpeakrate)
1882                         return (rack->rc_tp->t_maxpeakrate);
1883 #endif
1884                 /* Ok lets come up with the IW guess, if we have a srtt */
1885                 if (rack->rc_tp->t_srtt == 0) {
1886                         /*
1887                          * Go with old pacing method
1888                          * i.e. burst mitigation only.
1889                          */
1890                         return (0);
1891                 }
1892                 /* Ok lets get the initial TCP win (not racks) */
1893                 bw = tcp_compute_initwnd(tcp_maxseg(rack->rc_tp));
1894                 srtt = (uint64_t)rack->rc_tp->t_srtt;
1895                 bw *= (uint64_t)USECS_IN_SECOND;
1896                 bw /= srtt;
1897                 if (rack->r_ctl.bw_rate_cap && (bw > rack->r_ctl.bw_rate_cap))
1898                         bw = rack->r_ctl.bw_rate_cap;
1899                 return (bw);
1900         } else {
1901                 uint64_t bw;
1902
1903                 if (rack->r_ctl.num_measurements >= RACK_REQ_AVG) {
1904                         /* Averaging is done, we can return the value */
1905                         bw = rack->r_ctl.gp_bw;
1906                 } else {
1907                         /* Still doing initial average must calculate */
1908                         bw = rack->r_ctl.gp_bw / rack->r_ctl.num_measurements;
1909                 }
1910 #ifdef NETFLIX_PEAKRATE
1911                 if ((rack->rc_tp->t_maxpeakrate) &&
1912                     (bw > rack->rc_tp->t_maxpeakrate)) {
1913                         /* The user has set a peak rate to pace at
1914                          * don't allow us to pace faster than that.
1915                          */
1916                         return (rack->rc_tp->t_maxpeakrate);
1917                 }
1918 #endif
1919                 if (rack->r_ctl.bw_rate_cap && (bw > rack->r_ctl.bw_rate_cap))
1920                         bw = rack->r_ctl.bw_rate_cap;
1921                 return (bw);
1922         }
1923 }
1924
1925 static uint16_t
1926 rack_get_output_gain(struct tcp_rack *rack, struct rack_sendmap *rsm)
1927 {
1928         if (rack->use_fixed_rate) {
1929                 return (100);
1930         } else if (rack->in_probe_rtt && (rsm == NULL))
1931                 return (rack->r_ctl.rack_per_of_gp_probertt);
1932         else if ((IN_FASTRECOVERY(rack->rc_tp->t_flags) &&
1933                   rack->r_ctl.rack_per_of_gp_rec)) {
1934                 if (rsm) {
1935                         /* a retransmission always use the recovery rate */
1936                         return (rack->r_ctl.rack_per_of_gp_rec);
1937                 } else if (rack->rack_rec_nonrxt_use_cr) {
1938                         /* Directed to use the configured rate */
1939                         goto configured_rate;
1940                 } else if (rack->rack_no_prr &&
1941                            (rack->r_ctl.rack_per_of_gp_rec > 100)) {
1942                         /* No PRR, lets just use the b/w estimate only */
1943                         return (100);
1944                 } else {
1945                         /*
1946                          * Here we may have a non-retransmit but we
1947                          * have no overrides, so just use the recovery
1948                          * rate (prr is in effect).
1949                          */
1950                         return (rack->r_ctl.rack_per_of_gp_rec);
1951                 }
1952         }
1953 configured_rate:
1954         /* For the configured rate we look at our cwnd vs the ssthresh */
1955         if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh)
1956                 return (rack->r_ctl.rack_per_of_gp_ss);
1957         else
1958                 return (rack->r_ctl.rack_per_of_gp_ca);
1959 }
1960
1961 static void
1962 rack_log_dsack_event(struct tcp_rack *rack, uint8_t mod, uint32_t flex4, uint32_t flex5, uint32_t flex6)
1963 {
1964         /*
1965          * Types of logs (mod value)
1966          * 1 = dsack_persists reduced by 1 via T-O or fast recovery exit.
1967          * 2 = a dsack round begins, persist is reset to 16.
1968          * 3 = a dsack round ends
1969          * 4 = Dsack option increases rack rtt flex5 is the srtt input, flex6 is thresh
1970          * 5 = Socket option set changing the control flags rc_rack_tmr_std_based, rc_rack_use_dsack
1971          * 6 = Final rack rtt, flex4 is srtt and flex6 is final limited thresh.
1972          */
1973         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1974                 union tcp_log_stackspecific log;
1975                 struct timeval tv;
1976
1977                 memset(&log, 0, sizeof(log));
1978                 log.u_bbr.flex1 = rack->rc_rack_tmr_std_based;
1979                 log.u_bbr.flex1 <<= 1;
1980                 log.u_bbr.flex1 |= rack->rc_rack_use_dsack;
1981                 log.u_bbr.flex1 <<= 1;
1982                 log.u_bbr.flex1 |= rack->rc_dsack_round_seen;
1983                 log.u_bbr.flex2 = rack->r_ctl.dsack_round_end;
1984                 log.u_bbr.flex3 = rack->r_ctl.num_dsack;
1985                 log.u_bbr.flex4 = flex4;
1986                 log.u_bbr.flex5 = flex5;
1987                 log.u_bbr.flex6 = flex6;
1988                 log.u_bbr.flex7 = rack->r_ctl.dsack_persist;
1989                 log.u_bbr.flex8 = mod;
1990                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
1991                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
1992                     &rack->rc_inp->inp_socket->so_rcv,
1993                     &rack->rc_inp->inp_socket->so_snd,
1994                     RACK_DSACK_HANDLING, 0,
1995                     0, &log, false, &tv);
1996         }
1997 }
1998
1999 static void
2000 rack_log_hdwr_pacing(struct tcp_rack *rack,
2001                      uint64_t rate, uint64_t hw_rate, int line,
2002                      int error, uint16_t mod)
2003 {
2004         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2005                 union tcp_log_stackspecific log;
2006                 struct timeval tv;
2007                 const struct ifnet *ifp;
2008
2009                 memset(&log, 0, sizeof(log));
2010                 log.u_bbr.flex1 = ((hw_rate >> 32) & 0x00000000ffffffff);
2011                 log.u_bbr.flex2 = (hw_rate & 0x00000000ffffffff);
2012                 if (rack->r_ctl.crte) {
2013                         ifp = rack->r_ctl.crte->ptbl->rs_ifp;
2014                 } else if (rack->rc_inp->inp_route.ro_nh &&
2015                            rack->rc_inp->inp_route.ro_nh->nh_ifp) {
2016                         ifp = rack->rc_inp->inp_route.ro_nh->nh_ifp;
2017                 } else
2018                         ifp = NULL;
2019                 if (ifp) {
2020                         log.u_bbr.flex3 = (((uint64_t)ifp  >> 32) & 0x00000000ffffffff);
2021                         log.u_bbr.flex4 = ((uint64_t)ifp & 0x00000000ffffffff);
2022                 }
2023                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2024                 log.u_bbr.bw_inuse = rate;
2025                 log.u_bbr.flex5 = line;
2026                 log.u_bbr.flex6 = error;
2027                 log.u_bbr.flex7 = mod;
2028                 log.u_bbr.applimited = rack->r_ctl.rc_pace_max_segs;
2029                 log.u_bbr.flex8 = rack->use_fixed_rate;
2030                 log.u_bbr.flex8 <<= 1;
2031                 log.u_bbr.flex8 |= rack->rack_hdrw_pacing;
2032                 log.u_bbr.pkts_out = rack->rc_tp->t_maxseg;
2033                 log.u_bbr.delRate = rack->r_ctl.crte_prev_rate;
2034                 if (rack->r_ctl.crte)
2035                         log.u_bbr.cur_del_rate = rack->r_ctl.crte->rate;
2036                 else
2037                         log.u_bbr.cur_del_rate = 0;
2038                 log.u_bbr.rttProp = rack->r_ctl.last_hw_bw_req;
2039                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2040                     &rack->rc_inp->inp_socket->so_rcv,
2041                     &rack->rc_inp->inp_socket->so_snd,
2042                     BBR_LOG_HDWR_PACE, 0,
2043                     0, &log, false, &tv);
2044         }
2045 }
2046
2047 static uint64_t
2048 rack_get_output_bw(struct tcp_rack *rack, uint64_t bw, struct rack_sendmap *rsm, int *capped)
2049 {
2050         /*
2051          * We allow rack_per_of_gp_xx to dictate our bw rate we want.
2052          */
2053         uint64_t bw_est, high_rate;
2054         uint64_t gain;
2055
2056         gain = (uint64_t)rack_get_output_gain(rack, rsm);
2057         bw_est = bw * gain;
2058         bw_est /= (uint64_t)100;
2059         /* Never fall below the minimum (def 64kbps) */
2060         if (bw_est < RACK_MIN_BW)
2061                 bw_est = RACK_MIN_BW;
2062         if (rack->r_rack_hw_rate_caps) {
2063                 /* Rate caps are in place */
2064                 if (rack->r_ctl.crte != NULL) {
2065                         /* We have a hdwr rate already */
2066                         high_rate = tcp_hw_highest_rate(rack->r_ctl.crte);
2067                         if (bw_est >= high_rate) {
2068                                 /* We are capping bw at the highest rate table entry */
2069                                 rack_log_hdwr_pacing(rack,
2070                                                      bw_est, high_rate, __LINE__,
2071                                                      0, 3);
2072                                 bw_est = high_rate;
2073                                 if (capped)
2074                                         *capped = 1;
2075                         }
2076                 } else if ((rack->rack_hdrw_pacing == 0) &&
2077                            (rack->rack_hdw_pace_ena) &&
2078                            (rack->rack_attempt_hdwr_pace == 0) &&
2079                            (rack->rc_inp->inp_route.ro_nh != NULL) &&
2080                            (rack->rc_inp->inp_route.ro_nh->nh_ifp != NULL)) {
2081                         /*
2082                          * Special case, we have not yet attempted hardware
2083                          * pacing, and yet we may, when we do, find out if we are
2084                          * above the highest rate. We need to know the maxbw for the interface
2085                          * in question (if it supports ratelimiting). We get back
2086                          * a 0, if the interface is not found in the RL lists.
2087                          */
2088                         high_rate = tcp_hw_highest_rate_ifp(rack->rc_inp->inp_route.ro_nh->nh_ifp, rack->rc_inp);
2089                         if (high_rate) {
2090                                 /* Yep, we have a rate is it above this rate? */
2091                                 if (bw_est > high_rate) {
2092                                         bw_est = high_rate;
2093                                         if (capped)
2094                                                 *capped = 1;
2095                                 }
2096                         }
2097                 }
2098         }
2099         return (bw_est);
2100 }
2101
2102 static void
2103 rack_log_retran_reason(struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t tsused, uint32_t thresh, int mod)
2104 {
2105         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2106                 union tcp_log_stackspecific log;
2107                 struct timeval tv;
2108
2109                 if ((mod != 1) && (rack_verbose_logging == 0)) {
2110                         /*
2111                          * We get 3 values currently for mod
2112                          * 1 - We are retransmitting and this tells the reason.
2113                          * 2 - We are clearing a dup-ack count.
2114                          * 3 - We are incrementing a dup-ack count.
2115                          *
2116                          * The clear/increment are only logged
2117                          * if you have BBverbose on.
2118                          */
2119                         return;
2120                 }
2121                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2122                 log.u_bbr.flex1 = tsused;
2123                 log.u_bbr.flex2 = thresh;
2124                 log.u_bbr.flex3 = rsm->r_flags;
2125                 log.u_bbr.flex4 = rsm->r_dupack;
2126                 log.u_bbr.flex5 = rsm->r_start;
2127                 log.u_bbr.flex6 = rsm->r_end;
2128                 log.u_bbr.flex8 = mod;
2129                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2130                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2131                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2132                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2133                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2134                 log.u_bbr.pacing_gain = rack->r_must_retran;
2135                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2136                     &rack->rc_inp->inp_socket->so_rcv,
2137                     &rack->rc_inp->inp_socket->so_snd,
2138                     BBR_LOG_SETTINGS_CHG, 0,
2139                     0, &log, false, &tv);
2140         }
2141 }
2142
2143 static void
2144 rack_log_to_start(struct tcp_rack *rack, uint32_t cts, uint32_t to, int32_t slot, uint8_t which)
2145 {
2146         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2147                 union tcp_log_stackspecific log;
2148                 struct timeval tv;
2149
2150                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2151                 log.u_bbr.flex1 = rack->rc_tp->t_srtt;
2152                 log.u_bbr.flex2 = to;
2153                 log.u_bbr.flex3 = rack->r_ctl.rc_hpts_flags;
2154                 log.u_bbr.flex4 = slot;
2155                 log.u_bbr.flex5 = rack->rc_inp->inp_hptsslot;
2156                 log.u_bbr.flex6 = rack->rc_tp->t_rxtcur;
2157                 log.u_bbr.flex7 = rack->rc_in_persist;
2158                 log.u_bbr.flex8 = which;
2159                 if (rack->rack_no_prr)
2160                         log.u_bbr.pkts_out = 0;
2161                 else
2162                         log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
2163                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2164                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2165                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2166                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2167                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2168                 log.u_bbr.pacing_gain = rack->r_must_retran;
2169                 log.u_bbr.cwnd_gain = rack->rc_has_collapsed;
2170                 log.u_bbr.lt_epoch = rack->rc_tp->t_rxtshift;
2171                 log.u_bbr.lost = rack_rto_min;
2172                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2173                     &rack->rc_inp->inp_socket->so_rcv,
2174                     &rack->rc_inp->inp_socket->so_snd,
2175                     BBR_LOG_TIMERSTAR, 0,
2176                     0, &log, false, &tv);
2177         }
2178 }
2179
2180 static void
2181 rack_log_to_event(struct tcp_rack *rack, int32_t to_num, struct rack_sendmap *rsm)
2182 {
2183         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2184                 union tcp_log_stackspecific log;
2185                 struct timeval tv;
2186
2187                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2188                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2189                 log.u_bbr.flex8 = to_num;
2190                 log.u_bbr.flex1 = rack->r_ctl.rc_rack_min_rtt;
2191                 log.u_bbr.flex2 = rack->rc_rack_rtt;
2192                 if (rsm == NULL)
2193                         log.u_bbr.flex3 = 0;
2194                 else
2195                         log.u_bbr.flex3 = rsm->r_end - rsm->r_start;
2196                 if (rack->rack_no_prr)
2197                         log.u_bbr.flex5 = 0;
2198                 else
2199                         log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt;
2200                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2201                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2202                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2203                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2204                 log.u_bbr.pacing_gain = rack->r_must_retran;
2205                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2206                     &rack->rc_inp->inp_socket->so_rcv,
2207                     &rack->rc_inp->inp_socket->so_snd,
2208                     BBR_LOG_RTO, 0,
2209                     0, &log, false, &tv);
2210         }
2211 }
2212
2213 static void
2214 rack_log_map_chg(struct tcpcb *tp, struct tcp_rack *rack,
2215                  struct rack_sendmap *prev,
2216                  struct rack_sendmap *rsm,
2217                  struct rack_sendmap *next,
2218                  int flag, uint32_t th_ack, int line)
2219 {
2220         if (rack_verbose_logging && (tp->t_logstate != TCP_LOG_STATE_OFF)) {
2221                 union tcp_log_stackspecific log;
2222                 struct timeval tv;
2223
2224                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2225                 log.u_bbr.flex8 = flag;
2226                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2227                 log.u_bbr.cur_del_rate = (uint64_t)prev;
2228                 log.u_bbr.delRate = (uint64_t)rsm;
2229                 log.u_bbr.rttProp = (uint64_t)next;
2230                 log.u_bbr.flex7 = 0;
2231                 if (prev) {
2232                         log.u_bbr.flex1 = prev->r_start;
2233                         log.u_bbr.flex2 = prev->r_end;
2234                         log.u_bbr.flex7 |= 0x4;
2235                 }
2236                 if (rsm) {
2237                         log.u_bbr.flex3 = rsm->r_start;
2238                         log.u_bbr.flex4 = rsm->r_end;
2239                         log.u_bbr.flex7 |= 0x2;
2240                 }
2241                 if (next) {
2242                         log.u_bbr.flex5 = next->r_start;
2243                         log.u_bbr.flex6 = next->r_end;
2244                         log.u_bbr.flex7 |= 0x1;
2245                 }
2246                 log.u_bbr.applimited = line;
2247                 log.u_bbr.pkts_out = th_ack;
2248                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2249                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2250                 if (rack->rack_no_prr)
2251                         log.u_bbr.lost = 0;
2252                 else
2253                         log.u_bbr.lost = rack->r_ctl.rc_prr_sndcnt;
2254                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2255                     &rack->rc_inp->inp_socket->so_rcv,
2256                     &rack->rc_inp->inp_socket->so_snd,
2257                     TCP_LOG_MAPCHG, 0,
2258                     0, &log, false, &tv);
2259         }
2260 }
2261
2262 static void
2263 rack_log_rtt_upd(struct tcpcb *tp, struct tcp_rack *rack, uint32_t t, uint32_t len,
2264                  struct rack_sendmap *rsm, int conf)
2265 {
2266         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
2267                 union tcp_log_stackspecific log;
2268                 struct timeval tv;
2269                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2270                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2271                 log.u_bbr.flex1 = t;
2272                 log.u_bbr.flex2 = len;
2273                 log.u_bbr.flex3 = rack->r_ctl.rc_rack_min_rtt;
2274                 log.u_bbr.flex4 = rack->r_ctl.rack_rs.rs_rtt_lowest;
2275                 log.u_bbr.flex5 = rack->r_ctl.rack_rs.rs_rtt_highest;
2276                 log.u_bbr.flex6 = rack->r_ctl.rack_rs.rs_us_rtrcnt;
2277                 log.u_bbr.flex7 = conf;
2278                 log.u_bbr.rttProp = (uint64_t)rack->r_ctl.rack_rs.rs_rtt_tot;
2279                 log.u_bbr.flex8 = rack->r_ctl.rc_rate_sample_method;
2280                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2281                 log.u_bbr.delivered = rack->r_ctl.rack_rs.rs_us_rtrcnt;
2282                 log.u_bbr.pkts_out = rack->r_ctl.rack_rs.rs_flags;
2283                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2284                 if (rsm) {
2285                         log.u_bbr.pkt_epoch = rsm->r_start;
2286                         log.u_bbr.lost = rsm->r_end;
2287                         log.u_bbr.cwnd_gain = rsm->r_rtr_cnt;
2288                         /* We loose any upper of the 24 bits */
2289                         log.u_bbr.pacing_gain = (uint16_t)rsm->r_flags;
2290                 } else {
2291                         /* Its a SYN */
2292                         log.u_bbr.pkt_epoch = rack->rc_tp->iss;
2293                         log.u_bbr.lost = 0;
2294                         log.u_bbr.cwnd_gain = 0;
2295                         log.u_bbr.pacing_gain = 0;
2296                 }
2297                 /* Write out general bits of interest rrs here */
2298                 log.u_bbr.use_lt_bw = rack->rc_highly_buffered;
2299                 log.u_bbr.use_lt_bw <<= 1;
2300                 log.u_bbr.use_lt_bw |= rack->forced_ack;
2301                 log.u_bbr.use_lt_bw <<= 1;
2302                 log.u_bbr.use_lt_bw |= rack->rc_gp_dyn_mul;
2303                 log.u_bbr.use_lt_bw <<= 1;
2304                 log.u_bbr.use_lt_bw |= rack->in_probe_rtt;
2305                 log.u_bbr.use_lt_bw <<= 1;
2306                 log.u_bbr.use_lt_bw |= rack->measure_saw_probe_rtt;
2307                 log.u_bbr.use_lt_bw <<= 1;
2308                 log.u_bbr.use_lt_bw |= rack->app_limited_needs_set;
2309                 log.u_bbr.use_lt_bw <<= 1;
2310                 log.u_bbr.use_lt_bw |= rack->rc_gp_filled;
2311                 log.u_bbr.use_lt_bw <<= 1;
2312                 log.u_bbr.use_lt_bw |= rack->rc_dragged_bottom;
2313                 log.u_bbr.applimited = rack->r_ctl.rc_target_probertt_flight;
2314                 log.u_bbr.epoch = rack->r_ctl.rc_time_probertt_starts;
2315                 log.u_bbr.lt_epoch = rack->r_ctl.rc_time_probertt_entered;
2316                 log.u_bbr.cur_del_rate = rack->r_ctl.rc_lower_rtt_us_cts;
2317                 log.u_bbr.delRate = rack->r_ctl.rc_gp_srtt;
2318                 log.u_bbr.bw_inuse = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time);
2319                 log.u_bbr.bw_inuse <<= 32;
2320                 if (rsm)
2321                         log.u_bbr.bw_inuse |= ((uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]);
2322                 TCP_LOG_EVENTP(tp, NULL,
2323                     &rack->rc_inp->inp_socket->so_rcv,
2324                     &rack->rc_inp->inp_socket->so_snd,
2325                     BBR_LOG_BBRRTT, 0,
2326                     0, &log, false, &tv);
2327
2328
2329         }
2330 }
2331
2332 static void
2333 rack_log_rtt_sample(struct tcp_rack *rack, uint32_t rtt)
2334 {
2335         /*
2336          * Log the rtt sample we are
2337          * applying to the srtt algorithm in
2338          * useconds.
2339          */
2340         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2341                 union tcp_log_stackspecific log;
2342                 struct timeval tv;
2343
2344                 /* Convert our ms to a microsecond */
2345                 memset(&log, 0, sizeof(log));
2346                 log.u_bbr.flex1 = rtt;
2347                 log.u_bbr.flex2 = rack->r_ctl.ack_count;
2348                 log.u_bbr.flex3 = rack->r_ctl.sack_count;
2349                 log.u_bbr.flex4 = rack->r_ctl.sack_noextra_move;
2350                 log.u_bbr.flex5 = rack->r_ctl.sack_moved_extra;
2351                 log.u_bbr.flex6 = rack->rc_tp->t_rxtcur;
2352                 log.u_bbr.flex7 = 1;
2353                 log.u_bbr.flex8 = rack->sack_attack_disable;
2354                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2355                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2356                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2357                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2358                 log.u_bbr.pacing_gain = rack->r_must_retran;
2359                 /*
2360                  * We capture in delRate the upper 32 bits as
2361                  * the confidence level we had declared, and the
2362                  * lower 32 bits as the actual RTT using the arrival
2363                  * timestamp.
2364                  */
2365                 log.u_bbr.delRate = rack->r_ctl.rack_rs.confidence;
2366                 log.u_bbr.delRate <<= 32;
2367                 log.u_bbr.delRate |= rack->r_ctl.rack_rs.rs_us_rtt;
2368                 /* Lets capture all the things that make up t_rtxcur */
2369                 log.u_bbr.applimited = rack_rto_min;
2370                 log.u_bbr.epoch = rack_rto_max;
2371                 log.u_bbr.lt_epoch = rack->r_ctl.timer_slop;
2372                 log.u_bbr.lost = rack_rto_min;
2373                 log.u_bbr.pkt_epoch = TICKS_2_USEC(tcp_rexmit_slop);
2374                 log.u_bbr.rttProp = RACK_REXMTVAL(rack->rc_tp);
2375                 log.u_bbr.bw_inuse = rack->r_ctl.act_rcv_time.tv_sec;
2376                 log.u_bbr.bw_inuse *= HPTS_USEC_IN_SEC;
2377                 log.u_bbr.bw_inuse += rack->r_ctl.act_rcv_time.tv_usec;
2378                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2379                     &rack->rc_inp->inp_socket->so_rcv,
2380                     &rack->rc_inp->inp_socket->so_snd,
2381                     TCP_LOG_RTT, 0,
2382                     0, &log, false, &tv);
2383         }
2384 }
2385
2386 static void
2387 rack_log_rtt_sample_calc(struct tcp_rack *rack, uint32_t rtt, uint32_t send_time, uint32_t ack_time, int where)
2388 {
2389         if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2390                 union tcp_log_stackspecific log;
2391                 struct timeval tv;
2392
2393                 /* Convert our ms to a microsecond */
2394                 memset(&log, 0, sizeof(log));
2395                 log.u_bbr.flex1 = rtt;
2396                 log.u_bbr.flex2 = send_time;
2397                 log.u_bbr.flex3 = ack_time;
2398                 log.u_bbr.flex4 = where;
2399                 log.u_bbr.flex7 = 2;
2400                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2401                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2402                     &rack->rc_inp->inp_socket->so_rcv,
2403                     &rack->rc_inp->inp_socket->so_snd,
2404                     TCP_LOG_RTT, 0,
2405                     0, &log, false, &tv);
2406         }
2407 }
2408
2409
2410
2411 static inline void
2412 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick,  int event, int line)
2413 {
2414         if (rack_verbose_logging && (tp->t_logstate != TCP_LOG_STATE_OFF)) {
2415                 union tcp_log_stackspecific log;
2416                 struct timeval tv;
2417
2418                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2419                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2420                 log.u_bbr.flex1 = line;
2421                 log.u_bbr.flex2 = tick;
2422                 log.u_bbr.flex3 = tp->t_maxunacktime;
2423                 log.u_bbr.flex4 = tp->t_acktime;
2424                 log.u_bbr.flex8 = event;
2425                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2426                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2427                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2428                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2429                 log.u_bbr.pacing_gain = rack->r_must_retran;
2430                 TCP_LOG_EVENTP(tp, NULL,
2431                     &rack->rc_inp->inp_socket->so_rcv,
2432                     &rack->rc_inp->inp_socket->so_snd,
2433                     BBR_LOG_PROGRESS, 0,
2434                     0, &log, false, &tv);
2435         }
2436 }
2437
2438 static void
2439 rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t slot, uint32_t cts, struct timeval *tv)
2440 {
2441         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2442                 union tcp_log_stackspecific log;
2443
2444                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2445                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2446                 log.u_bbr.flex1 = slot;
2447                 if (rack->rack_no_prr)
2448                         log.u_bbr.flex2 = 0;
2449                 else
2450                         log.u_bbr.flex2 = rack->r_ctl.rc_prr_sndcnt;
2451                 log.u_bbr.flex7 = (0x0000ffff & rack->r_ctl.rc_hpts_flags);
2452                 log.u_bbr.flex8 = rack->rc_in_persist;
2453                 log.u_bbr.timeStamp = cts;
2454                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2455                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2456                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2457                 log.u_bbr.pacing_gain = rack->r_must_retran;
2458                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2459                     &rack->rc_inp->inp_socket->so_rcv,
2460                     &rack->rc_inp->inp_socket->so_snd,
2461                     BBR_LOG_BBRSND, 0,
2462                     0, &log, false, tv);
2463         }
2464 }
2465
2466 static void
2467 rack_log_doseg_done(struct tcp_rack *rack, uint32_t cts, int32_t nxt_pkt, int32_t did_out, int way_out, int nsegs)
2468 {
2469         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2470                 union tcp_log_stackspecific log;
2471                 struct timeval tv;
2472
2473                 memset(&log, 0, sizeof(log));
2474                 log.u_bbr.flex1 = did_out;
2475                 log.u_bbr.flex2 = nxt_pkt;
2476                 log.u_bbr.flex3 = way_out;
2477                 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags;
2478                 if (rack->rack_no_prr)
2479                         log.u_bbr.flex5 = 0;
2480                 else
2481                         log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt;
2482                 log.u_bbr.flex6 = nsegs;
2483                 log.u_bbr.applimited = rack->r_ctl.rc_pace_min_segs;
2484                 log.u_bbr.flex7 = rack->rc_ack_can_sendout_data;        /* Do we have ack-can-send set */
2485                 log.u_bbr.flex7 <<= 1;
2486                 log.u_bbr.flex7 |= rack->r_fast_output; /* is fast output primed */
2487                 log.u_bbr.flex7 <<= 1;
2488                 log.u_bbr.flex7 |= rack->r_wanted_output;       /* Do we want output */
2489                 log.u_bbr.flex8 = rack->rc_in_persist;
2490                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2491                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2492                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2493                 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
2494                 log.u_bbr.use_lt_bw <<= 1;
2495                 log.u_bbr.use_lt_bw |= rack->r_might_revert;
2496                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2497                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2498                 log.u_bbr.pacing_gain = rack->r_must_retran;
2499                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2500                     &rack->rc_inp->inp_socket->so_rcv,
2501                     &rack->rc_inp->inp_socket->so_snd,
2502                     BBR_LOG_DOSEG_DONE, 0,
2503                     0, &log, false, &tv);
2504         }
2505 }
2506
2507 static void
2508 rack_log_type_pacing_sizes(struct tcpcb *tp, struct tcp_rack *rack, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint8_t frm)
2509 {
2510         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
2511                 union tcp_log_stackspecific log;
2512                 struct timeval tv;
2513
2514                 memset(&log, 0, sizeof(log));
2515                 log.u_bbr.flex1 = rack->r_ctl.rc_pace_min_segs;
2516                 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs;
2517                 log.u_bbr.flex4 = arg1;
2518                 log.u_bbr.flex5 = arg2;
2519                 log.u_bbr.flex6 = arg3;
2520                 log.u_bbr.flex8 = frm;
2521                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2522                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2523                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2524                 log.u_bbr.applimited = rack->r_ctl.rc_sacked;
2525                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2526                 log.u_bbr.pacing_gain = rack->r_must_retran;
2527                 TCP_LOG_EVENTP(tp, NULL,
2528                     &tp->t_inpcb->inp_socket->so_rcv,
2529                     &tp->t_inpcb->inp_socket->so_snd,
2530                     TCP_HDWR_PACE_SIZE, 0,
2531                     0, &log, false, &tv);
2532         }
2533 }
2534
2535 static void
2536 rack_log_type_just_return(struct tcp_rack *rack, uint32_t cts, uint32_t tlen, uint32_t slot,
2537                           uint8_t hpts_calling, int reason, uint32_t cwnd_to_use)
2538 {
2539         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2540                 union tcp_log_stackspecific log;
2541                 struct timeval tv;
2542
2543                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2544                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2545                 log.u_bbr.flex1 = slot;
2546                 log.u_bbr.flex2 = rack->r_ctl.rc_hpts_flags;
2547                 log.u_bbr.flex4 = reason;
2548                 if (rack->rack_no_prr)
2549                         log.u_bbr.flex5 = 0;
2550                 else
2551                         log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt;
2552                 log.u_bbr.flex7 = hpts_calling;
2553                 log.u_bbr.flex8 = rack->rc_in_persist;
2554                 log.u_bbr.lt_epoch = cwnd_to_use;
2555                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2556                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2557                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2558                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2559                 log.u_bbr.pacing_gain = rack->r_must_retran;
2560                 log.u_bbr.cwnd_gain = rack->rc_has_collapsed;
2561                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2562                     &rack->rc_inp->inp_socket->so_rcv,
2563                     &rack->rc_inp->inp_socket->so_snd,
2564                     BBR_LOG_JUSTRET, 0,
2565                     tlen, &log, false, &tv);
2566         }
2567 }
2568
2569 static void
2570 rack_log_to_cancel(struct tcp_rack *rack, int32_t hpts_removed, int line, uint32_t us_cts,
2571                    struct timeval *tv, uint32_t flags_on_entry)
2572 {
2573         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2574                 union tcp_log_stackspecific log;
2575
2576                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2577                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
2578                 log.u_bbr.flex1 = line;
2579                 log.u_bbr.flex2 = rack->r_ctl.rc_last_output_to;
2580                 log.u_bbr.flex3 = flags_on_entry;
2581                 log.u_bbr.flex4 = us_cts;
2582                 if (rack->rack_no_prr)
2583                         log.u_bbr.flex5 = 0;
2584                 else
2585                         log.u_bbr.flex5 = rack->r_ctl.rc_prr_sndcnt;
2586                 log.u_bbr.flex6 = rack->rc_tp->t_rxtcur;
2587                 log.u_bbr.flex7 = hpts_removed;
2588                 log.u_bbr.flex8 = 1;
2589                 log.u_bbr.applimited = rack->r_ctl.rc_hpts_flags;
2590                 log.u_bbr.timeStamp = us_cts;
2591                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2592                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2593                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2594                 log.u_bbr.pacing_gain = rack->r_must_retran;
2595                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2596                     &rack->rc_inp->inp_socket->so_rcv,
2597                     &rack->rc_inp->inp_socket->so_snd,
2598                     BBR_LOG_TIMERCANC, 0,
2599                     0, &log, false, tv);
2600         }
2601 }
2602
2603 static void
2604 rack_log_alt_to_to_cancel(struct tcp_rack *rack,
2605                           uint32_t flex1, uint32_t flex2,
2606                           uint32_t flex3, uint32_t flex4,
2607                           uint32_t flex5, uint32_t flex6,
2608                           uint16_t flex7, uint8_t mod)
2609 {
2610         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2611                 union tcp_log_stackspecific log;
2612                 struct timeval tv;
2613
2614                 if (mod == 1) {
2615                         /* No you can't use 1, its for the real to cancel */
2616                         return;
2617                 }
2618                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2619                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2620                 log.u_bbr.flex1 = flex1;
2621                 log.u_bbr.flex2 = flex2;
2622                 log.u_bbr.flex3 = flex3;
2623                 log.u_bbr.flex4 = flex4;
2624                 log.u_bbr.flex5 = flex5;
2625                 log.u_bbr.flex6 = flex6;
2626                 log.u_bbr.flex7 = flex7;
2627                 log.u_bbr.flex8 = mod;
2628                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2629                     &rack->rc_inp->inp_socket->so_rcv,
2630                     &rack->rc_inp->inp_socket->so_snd,
2631                     BBR_LOG_TIMERCANC, 0,
2632                     0, &log, false, &tv);
2633         }
2634 }
2635
2636 static void
2637 rack_log_to_processing(struct tcp_rack *rack, uint32_t cts, int32_t ret, int32_t timers)
2638 {
2639         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2640                 union tcp_log_stackspecific log;
2641                 struct timeval tv;
2642
2643                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2644                 log.u_bbr.flex1 = timers;
2645                 log.u_bbr.flex2 = ret;
2646                 log.u_bbr.flex3 = rack->r_ctl.rc_timer_exp;
2647                 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags;
2648                 log.u_bbr.flex5 = cts;
2649                 if (rack->rack_no_prr)
2650                         log.u_bbr.flex6 = 0;
2651                 else
2652                         log.u_bbr.flex6 = rack->r_ctl.rc_prr_sndcnt;
2653                 log.u_bbr.pkts_out = rack->r_ctl.rc_out_at_rto;
2654                 log.u_bbr.delivered = rack->r_ctl.rc_snd_max_at_rto;
2655                 log.u_bbr.pacing_gain = rack->r_must_retran;
2656                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2657                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2658                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2659                     &rack->rc_inp->inp_socket->so_rcv,
2660                     &rack->rc_inp->inp_socket->so_snd,
2661                     BBR_LOG_TO_PROCESS, 0,
2662                     0, &log, false, &tv);
2663         }
2664 }
2665
2666 static void
2667 rack_log_to_prr(struct tcp_rack *rack, int frm, int orig_cwnd, int line)
2668 {
2669         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2670                 union tcp_log_stackspecific log;
2671                 struct timeval tv;
2672
2673                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2674                 log.u_bbr.flex1 = rack->r_ctl.rc_prr_out;
2675                 log.u_bbr.flex2 = rack->r_ctl.rc_prr_recovery_fs;
2676                 if (rack->rack_no_prr)
2677                         log.u_bbr.flex3 = 0;
2678                 else
2679                         log.u_bbr.flex3 = rack->r_ctl.rc_prr_sndcnt;
2680                 log.u_bbr.flex4 = rack->r_ctl.rc_prr_delivered;
2681                 log.u_bbr.flex5 = rack->r_ctl.rc_sacked;
2682                 log.u_bbr.flex6 = rack->r_ctl.rc_holes_rxt;
2683                 log.u_bbr.flex7 = line;
2684                 log.u_bbr.flex8 = frm;
2685                 log.u_bbr.pkts_out = orig_cwnd;
2686                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2687                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2688                 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
2689                 log.u_bbr.use_lt_bw <<= 1;
2690                 log.u_bbr.use_lt_bw |= rack->r_might_revert;
2691                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2692                     &rack->rc_inp->inp_socket->so_rcv,
2693                     &rack->rc_inp->inp_socket->so_snd,
2694                     BBR_LOG_BBRUPD, 0,
2695                     0, &log, false, &tv);
2696         }
2697 }
2698
2699 #ifdef NETFLIX_EXP_DETECTION
2700 static void
2701 rack_log_sad(struct tcp_rack *rack, int event)
2702 {
2703         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2704                 union tcp_log_stackspecific log;
2705                 struct timeval tv;
2706
2707                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
2708                 log.u_bbr.flex1 = rack->r_ctl.sack_count;
2709                 log.u_bbr.flex2 = rack->r_ctl.ack_count;
2710                 log.u_bbr.flex3 = rack->r_ctl.sack_moved_extra;
2711                 log.u_bbr.flex4 = rack->r_ctl.sack_noextra_move;
2712                 log.u_bbr.flex5 = rack->r_ctl.rc_num_maps_alloced;
2713                 log.u_bbr.flex6 = tcp_sack_to_ack_thresh;
2714                 log.u_bbr.pkts_out = tcp_sack_to_move_thresh;
2715                 log.u_bbr.lt_epoch = (tcp_force_detection << 8);
2716                 log.u_bbr.lt_epoch |= rack->do_detection;
2717                 log.u_bbr.applimited = tcp_map_minimum;
2718                 log.u_bbr.flex7 = rack->sack_attack_disable;
2719                 log.u_bbr.flex8 = event;
2720                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
2721                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
2722                 log.u_bbr.delivered = tcp_sad_decay_val;
2723                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
2724                     &rack->rc_inp->inp_socket->so_rcv,
2725                     &rack->rc_inp->inp_socket->so_snd,
2726                     TCP_SAD_DETECTION, 0,
2727                     0, &log, false, &tv);
2728         }
2729 }
2730 #endif
2731
2732 static void
2733 rack_counter_destroy(void)
2734 {
2735         counter_u64_free(rack_fto_send);
2736         counter_u64_free(rack_fto_rsm_send);
2737         counter_u64_free(rack_nfto_resend);
2738         counter_u64_free(rack_hw_pace_init_fail);
2739         counter_u64_free(rack_hw_pace_lost);
2740         counter_u64_free(rack_non_fto_send);
2741         counter_u64_free(rack_extended_rfo);
2742         counter_u64_free(rack_ack_total);
2743         counter_u64_free(rack_express_sack);
2744         counter_u64_free(rack_sack_total);
2745         counter_u64_free(rack_move_none);
2746         counter_u64_free(rack_move_some);
2747         counter_u64_free(rack_sack_attacks_detected);
2748         counter_u64_free(rack_sack_attacks_reversed);
2749         counter_u64_free(rack_sack_used_next_merge);
2750         counter_u64_free(rack_sack_used_prev_merge);
2751         counter_u64_free(rack_tlp_tot);
2752         counter_u64_free(rack_tlp_newdata);
2753         counter_u64_free(rack_tlp_retran);
2754         counter_u64_free(rack_tlp_retran_bytes);
2755         counter_u64_free(rack_to_tot);
2756         counter_u64_free(rack_saw_enobuf);
2757         counter_u64_free(rack_saw_enobuf_hw);
2758         counter_u64_free(rack_saw_enetunreach);
2759         counter_u64_free(rack_hot_alloc);
2760         counter_u64_free(rack_to_alloc);
2761         counter_u64_free(rack_to_alloc_hard);
2762         counter_u64_free(rack_to_alloc_emerg);
2763         counter_u64_free(rack_to_alloc_limited);
2764         counter_u64_free(rack_alloc_limited_conns);
2765         counter_u64_free(rack_split_limited);
2766         counter_u64_free(rack_multi_single_eq);
2767         counter_u64_free(rack_proc_non_comp_ack);
2768         counter_u64_free(rack_sack_proc_all);
2769         counter_u64_free(rack_sack_proc_restart);
2770         counter_u64_free(rack_sack_proc_short);
2771         counter_u64_free(rack_sack_skipped_acked);
2772         counter_u64_free(rack_sack_splits);
2773         counter_u64_free(rack_input_idle_reduces);
2774         counter_u64_free(rack_collapsed_win);
2775         counter_u64_free(rack_try_scwnd);
2776         counter_u64_free(rack_persists_sends);
2777         counter_u64_free(rack_persists_acks);
2778         counter_u64_free(rack_persists_loss);
2779         counter_u64_free(rack_persists_lost_ends);
2780 #ifdef INVARIANTS
2781         counter_u64_free(rack_adjust_map_bw);
2782 #endif
2783         COUNTER_ARRAY_FREE(rack_out_size, TCP_MSS_ACCT_SIZE);
2784         COUNTER_ARRAY_FREE(rack_opts_arry, RACK_OPTS_SIZE);
2785 }
2786
2787 static struct rack_sendmap *
2788 rack_alloc(struct tcp_rack *rack)
2789 {
2790         struct rack_sendmap *rsm;
2791
2792         /*
2793          * First get the top of the list it in
2794          * theory is the "hottest" rsm we have,
2795          * possibly just freed by ack processing.
2796          */
2797         if (rack->rc_free_cnt > rack_free_cache) {
2798                 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free);
2799                 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext);
2800                 counter_u64_add(rack_hot_alloc, 1);
2801                 rack->rc_free_cnt--;
2802                 return (rsm);
2803         }
2804         /*
2805          * Once we get under our free cache we probably
2806          * no longer have a "hot" one available. Lets
2807          * get one from UMA.
2808          */
2809         rsm = uma_zalloc(rack_zone, M_NOWAIT);
2810         if (rsm) {
2811                 rack->r_ctl.rc_num_maps_alloced++;
2812                 counter_u64_add(rack_to_alloc, 1);
2813                 return (rsm);
2814         }
2815         /*
2816          * Dig in to our aux rsm's (the last two) since
2817          * UMA failed to get us one.
2818          */
2819         if (rack->rc_free_cnt) {
2820                 counter_u64_add(rack_to_alloc_emerg, 1);
2821                 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free);
2822                 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext);
2823                 rack->rc_free_cnt--;
2824                 return (rsm);
2825         }
2826         return (NULL);
2827 }
2828
2829 static struct rack_sendmap *
2830 rack_alloc_full_limit(struct tcp_rack *rack)
2831 {
2832         if ((V_tcp_map_entries_limit > 0) &&
2833             (rack->do_detection == 0) &&
2834             (rack->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
2835                 counter_u64_add(rack_to_alloc_limited, 1);
2836                 if (!rack->alloc_limit_reported) {
2837                         rack->alloc_limit_reported = 1;
2838                         counter_u64_add(rack_alloc_limited_conns, 1);
2839                 }
2840                 return (NULL);
2841         }
2842         return (rack_alloc(rack));
2843 }
2844
2845 /* wrapper to allocate a sendmap entry, subject to a specific limit */
2846 static struct rack_sendmap *
2847 rack_alloc_limit(struct tcp_rack *rack, uint8_t limit_type)
2848 {
2849         struct rack_sendmap *rsm;
2850
2851         if (limit_type) {
2852                 /* currently there is only one limit type */
2853                 if (V_tcp_map_split_limit > 0 &&
2854                     (rack->do_detection == 0) &&
2855                     rack->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) {
2856                         counter_u64_add(rack_split_limited, 1);
2857                         if (!rack->alloc_limit_reported) {
2858                                 rack->alloc_limit_reported = 1;
2859                                 counter_u64_add(rack_alloc_limited_conns, 1);
2860                         }
2861                         return (NULL);
2862                 }
2863         }
2864
2865         /* allocate and mark in the limit type, if set */
2866         rsm = rack_alloc(rack);
2867         if (rsm != NULL && limit_type) {
2868                 rsm->r_limit_type = limit_type;
2869                 rack->r_ctl.rc_num_split_allocs++;
2870         }
2871         return (rsm);
2872 }
2873
2874 static void
2875 rack_free(struct tcp_rack *rack, struct rack_sendmap *rsm)
2876 {
2877         if (rsm->r_flags & RACK_APP_LIMITED) {
2878                 if (rack->r_ctl.rc_app_limited_cnt > 0) {
2879                         rack->r_ctl.rc_app_limited_cnt--;
2880                 }
2881         }
2882         if (rsm->r_limit_type) {
2883                 /* currently there is only one limit type */
2884                 rack->r_ctl.rc_num_split_allocs--;
2885         }
2886         if (rsm == rack->r_ctl.rc_first_appl) {
2887                 if (rack->r_ctl.rc_app_limited_cnt == 0)
2888                         rack->r_ctl.rc_first_appl = NULL;
2889                 else {
2890                         /* Follow the next one out */
2891                         struct rack_sendmap fe;
2892
2893                         fe.r_start = rsm->r_nseq_appl;
2894                         rack->r_ctl.rc_first_appl = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe);
2895                 }
2896         }
2897         if (rsm == rack->r_ctl.rc_resend)
2898                 rack->r_ctl.rc_resend = NULL;
2899         if (rsm == rack->r_ctl.rc_end_appl)
2900                 rack->r_ctl.rc_end_appl = NULL;
2901         if (rack->r_ctl.rc_tlpsend == rsm)
2902                 rack->r_ctl.rc_tlpsend = NULL;
2903         if (rack->r_ctl.rc_sacklast == rsm)
2904                 rack->r_ctl.rc_sacklast = NULL;
2905         memset(rsm, 0, sizeof(struct rack_sendmap));
2906         TAILQ_INSERT_HEAD(&rack->r_ctl.rc_free, rsm, r_tnext);
2907         rack->rc_free_cnt++;
2908 }
2909
2910 static void
2911 rack_free_trim(struct tcp_rack *rack)
2912 {
2913         struct rack_sendmap *rsm;
2914
2915         /*
2916          * Free up all the tail entries until
2917          * we get our list down to the limit.
2918          */
2919         while (rack->rc_free_cnt > rack_free_cache) {
2920                 rsm = TAILQ_LAST(&rack->r_ctl.rc_free, rack_head);
2921                 TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext);
2922                 rack->rc_free_cnt--;
2923                 uma_zfree(rack_zone, rsm);
2924         }
2925 }
2926
2927
2928 static uint32_t
2929 rack_get_measure_window(struct tcpcb *tp, struct tcp_rack *rack)
2930 {
2931         uint64_t srtt, bw, len, tim;
2932         uint32_t segsiz, def_len, minl;
2933
2934         segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
2935         def_len = rack_def_data_window * segsiz;
2936         if (rack->rc_gp_filled == 0) {
2937                 /*
2938                  * We have no measurement (IW is in flight?) so
2939                  * we can only guess using our data_window sysctl
2940                  * value (usually 20MSS).
2941                  */
2942                 return (def_len);
2943         }
2944         /*
2945          * Now we have a number of factors to consider.
2946          *
2947          * 1) We have a desired BDP which is usually
2948          *    at least 2.
2949          * 2) We have a minimum number of rtt's usually 1 SRTT
2950          *    but we allow it too to be more.
2951          * 3) We want to make sure a measurement last N useconds (if
2952          *    we have set rack_min_measure_usec.
2953          *
2954          * We handle the first concern here by trying to create a data
2955          * window of max(rack_def_data_window, DesiredBDP). The
2956          * second concern we handle in not letting the measurement
2957          * window end normally until at least the required SRTT's
2958          * have gone by which is done further below in
2959          * rack_enough_for_measurement(). Finally the third concern
2960          * we also handle here by calculating how long that time
2961          * would take at the current BW and then return the
2962          * max of our first calculation and that length. Note
2963          * that if rack_min_measure_usec is 0, we don't deal
2964          * with concern 3. Also for both Concern 1 and 3 an
2965          * application limited period could end the measurement
2966          * earlier.
2967          *
2968          * So lets calculate the BDP with the "known" b/w using
2969          * the SRTT has our rtt and then multiply it by the
2970          * goal.
2971          */
2972         bw = rack_get_bw(rack);
2973         srtt = (uint64_t)tp->t_srtt;
2974         len = bw * srtt;
2975         len /= (uint64_t)HPTS_USEC_IN_SEC;
2976         len *= max(1, rack_goal_bdp);
2977         /* Now we need to round up to the nearest MSS */
2978         len = roundup(len, segsiz);
2979         if (rack_min_measure_usec) {
2980                 /* Now calculate our min length for this b/w */
2981                 tim = rack_min_measure_usec;
2982                 minl = (tim * bw) / (uint64_t)HPTS_USEC_IN_SEC;
2983                 if (minl == 0)
2984                         minl = 1;
2985                 minl = roundup(minl, segsiz);
2986                 if (len < minl)
2987                         len = minl;
2988         }
2989         /*
2990          * Now if we have a very small window we want
2991          * to attempt to get the window that is
2992          * as small as possible. This happens on
2993          * low b/w connections and we don't want to
2994          * span huge numbers of rtt's between measurements.
2995          *
2996          * We basically include 2 over our "MIN window" so
2997          * that the measurement can be shortened (possibly) by
2998          * an ack'ed packet.
2999          */
3000         if (len < def_len)
3001                 return (max((uint32_t)len, ((MIN_GP_WIN+2) * segsiz)));
3002         else
3003                 return (max((uint32_t)len, def_len));
3004
3005 }
3006
3007 static int
3008 rack_enough_for_measurement(struct tcpcb *tp, struct tcp_rack *rack, tcp_seq th_ack, uint8_t *quality)
3009 {
3010         uint32_t tim, srtts, segsiz;
3011
3012         /*
3013          * Has enough time passed for the GP measurement to be valid?
3014          */
3015         if ((tp->snd_max == tp->snd_una) ||
3016             (th_ack == tp->snd_max)){
3017                 /* All is acked */
3018                 *quality = RACK_QUALITY_ALLACKED;
3019                 return (1);
3020         }
3021         if (SEQ_LT(th_ack, tp->gput_seq)) {
3022                 /* Not enough bytes yet */
3023                 return (0);
3024         }
3025         segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
3026         if (SEQ_LT(th_ack, tp->gput_ack) &&
3027             ((th_ack - tp->gput_seq) < max(rc_init_window(rack), (MIN_GP_WIN * segsiz)))) {
3028                 /* Not enough bytes yet */
3029                 return (0);
3030         }
3031         if (rack->r_ctl.rc_first_appl &&
3032             (SEQ_GEQ(th_ack, rack->r_ctl.rc_first_appl->r_end))) {
3033                 /*
3034                  * We are up to the app limited send point
3035                  * we have to measure irrespective of the time..
3036                  */
3037                 *quality = RACK_QUALITY_APPLIMITED;
3038                 return (1);
3039         }
3040         /* Now what about time? */
3041         srtts = (rack->r_ctl.rc_gp_srtt * rack_min_srtts);
3042         tim = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time) - tp->gput_ts;
3043         if (tim >= srtts) {
3044                 *quality = RACK_QUALITY_HIGH;
3045                 return (1);
3046         }
3047         /* Nope not even a full SRTT has passed */
3048         return (0);
3049 }
3050
3051 static void
3052 rack_log_timely(struct tcp_rack *rack,
3053                 uint32_t logged, uint64_t cur_bw, uint64_t low_bnd,
3054                 uint64_t up_bnd, int line, uint8_t method)
3055 {
3056         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
3057                 union tcp_log_stackspecific log;
3058                 struct timeval tv;
3059
3060                 memset(&log, 0, sizeof(log));
3061                 log.u_bbr.flex1 = logged;
3062                 log.u_bbr.flex2 = rack->rc_gp_timely_inc_cnt;
3063                 log.u_bbr.flex2 <<= 4;
3064                 log.u_bbr.flex2 |= rack->rc_gp_timely_dec_cnt;
3065                 log.u_bbr.flex2 <<= 4;
3066                 log.u_bbr.flex2 |= rack->rc_gp_incr;
3067                 log.u_bbr.flex2 <<= 4;
3068                 log.u_bbr.flex2 |= rack->rc_gp_bwred;
3069                 log.u_bbr.flex3 = rack->rc_gp_incr;
3070                 log.u_bbr.flex4 = rack->r_ctl.rack_per_of_gp_ss;
3071                 log.u_bbr.flex5 = rack->r_ctl.rack_per_of_gp_ca;
3072                 log.u_bbr.flex6 = rack->r_ctl.rack_per_of_gp_rec;
3073                 log.u_bbr.flex7 = rack->rc_gp_bwred;
3074                 log.u_bbr.flex8 = method;
3075                 log.u_bbr.cur_del_rate = cur_bw;
3076                 log.u_bbr.delRate = low_bnd;
3077                 log.u_bbr.bw_inuse = up_bnd;
3078                 log.u_bbr.rttProp = rack_get_bw(rack);
3079                 log.u_bbr.pkt_epoch = line;
3080                 log.u_bbr.pkts_out = rack->r_ctl.rc_rtt_diff;
3081                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3082                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3083                 log.u_bbr.epoch = rack->r_ctl.rc_gp_srtt;
3084                 log.u_bbr.lt_epoch = rack->r_ctl.rc_prev_gp_srtt;
3085                 log.u_bbr.cwnd_gain = rack->rc_dragged_bottom;
3086                 log.u_bbr.cwnd_gain <<= 1;
3087                 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_rec;
3088                 log.u_bbr.cwnd_gain <<= 1;
3089                 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ss;
3090                 log.u_bbr.cwnd_gain <<= 1;
3091                 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ca;
3092                 log.u_bbr.lost = rack->r_ctl.rc_loss_count;
3093                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3094                     &rack->rc_inp->inp_socket->so_rcv,
3095                     &rack->rc_inp->inp_socket->so_snd,
3096                     TCP_TIMELY_WORK, 0,
3097                     0, &log, false, &tv);
3098         }
3099 }
3100
3101 static int
3102 rack_bw_can_be_raised(struct tcp_rack *rack, uint64_t cur_bw, uint64_t last_bw_est, uint16_t mult)
3103 {
3104         /*
3105          * Before we increase we need to know if
3106          * the estimate just made was less than
3107          * our pacing goal (i.e. (cur_bw * mult) > last_bw_est)
3108          *
3109          * If we already are pacing at a fast enough
3110          * rate to push us faster there is no sense of
3111          * increasing.
3112          *
3113          * We first caculate our actual pacing rate (ss or ca multiplier
3114          * times our cur_bw).
3115          *
3116          * Then we take the last measured rate and multipy by our
3117          * maximum pacing overage to give us a max allowable rate.
3118          *
3119          * If our act_rate is smaller than our max_allowable rate
3120          * then we should increase. Else we should hold steady.
3121          *
3122          */
3123         uint64_t act_rate, max_allow_rate;
3124
3125         if (rack_timely_no_stopping)
3126                 return (1);
3127
3128         if ((cur_bw == 0) || (last_bw_est == 0)) {
3129                 /*
3130                  * Initial startup case or
3131                  * everything is acked case.
3132                  */
3133                 rack_log_timely(rack,  mult, cur_bw, 0, 0,
3134                                 __LINE__, 9);
3135                 return (1);
3136         }
3137         if (mult <= 100) {
3138                 /*
3139                  * We can always pace at or slightly above our rate.
3140                  */
3141                 rack_log_timely(rack,  mult, cur_bw, 0, 0,
3142                                 __LINE__, 9);
3143                 return (1);
3144         }
3145         act_rate = cur_bw * (uint64_t)mult;
3146         act_rate /= 100;
3147         max_allow_rate = last_bw_est * ((uint64_t)rack_max_per_above + (uint64_t)100);
3148         max_allow_rate /= 100;
3149         if (act_rate < max_allow_rate) {
3150                 /*
3151                  * Here the rate we are actually pacing at
3152                  * is smaller than 10% above our last measurement.
3153                  * This means we are pacing below what we would
3154                  * like to try to achieve (plus some wiggle room).
3155                  */
3156                 rack_log_timely(rack,  mult, cur_bw, act_rate, max_allow_rate,
3157                                 __LINE__, 9);
3158                 return (1);
3159         } else {
3160                 /*
3161                  * Here we are already pacing at least rack_max_per_above(10%)
3162                  * what we are getting back. This indicates most likely
3163                  * that we are being limited (cwnd/rwnd/app) and can't
3164                  * get any more b/w. There is no sense of trying to
3165                  * raise up the pacing rate its not speeding us up
3166                  * and we already are pacing faster than we are getting.
3167                  */
3168                 rack_log_timely(rack,  mult, cur_bw, act_rate, max_allow_rate,
3169                                 __LINE__, 8);
3170                 return (0);
3171         }
3172 }
3173
3174 static void
3175 rack_validate_multipliers_at_or_above100(struct tcp_rack *rack)
3176 {
3177         /*
3178          * When we drag bottom, we want to assure
3179          * that no multiplier is below 1.0, if so
3180          * we want to restore it to at least that.
3181          */
3182         if (rack->r_ctl.rack_per_of_gp_rec  < 100) {
3183                 /* This is unlikely we usually do not touch recovery */
3184                 rack->r_ctl.rack_per_of_gp_rec = 100;
3185         }
3186         if (rack->r_ctl.rack_per_of_gp_ca < 100) {
3187                 rack->r_ctl.rack_per_of_gp_ca = 100;
3188         }
3189         if (rack->r_ctl.rack_per_of_gp_ss < 100) {
3190                 rack->r_ctl.rack_per_of_gp_ss = 100;
3191         }
3192 }
3193
3194 static void
3195 rack_validate_multipliers_at_or_below_100(struct tcp_rack *rack)
3196 {
3197         if (rack->r_ctl.rack_per_of_gp_ca > 100) {
3198                 rack->r_ctl.rack_per_of_gp_ca = 100;
3199         }
3200         if (rack->r_ctl.rack_per_of_gp_ss > 100) {
3201                 rack->r_ctl.rack_per_of_gp_ss = 100;
3202         }
3203 }
3204
3205 static void
3206 rack_increase_bw_mul(struct tcp_rack *rack, int timely_says, uint64_t cur_bw, uint64_t last_bw_est, int override)
3207 {
3208         int32_t  calc, logged, plus;
3209
3210         logged = 0;
3211
3212         if (override) {
3213                 /*
3214                  * override is passed when we are
3215                  * loosing b/w and making one last
3216                  * gasp at trying to not loose out
3217                  * to a new-reno flow.
3218                  */
3219                 goto extra_boost;
3220         }
3221         /* In classic timely we boost by 5x if we have 5 increases in a row, lets not */
3222         if (rack->rc_gp_incr &&
3223             ((rack->rc_gp_timely_inc_cnt + 1) >= RACK_TIMELY_CNT_BOOST)) {
3224                 /*
3225                  * Reset and get 5 strokes more before the boost. Note
3226                  * that the count is 0 based so we have to add one.
3227                  */
3228 extra_boost:
3229                 plus = (uint32_t)rack_gp_increase_per * RACK_TIMELY_CNT_BOOST;
3230                 rack->rc_gp_timely_inc_cnt = 0;
3231         } else
3232                 plus = (uint32_t)rack_gp_increase_per;
3233         /* Must be at least 1% increase for true timely increases */
3234         if ((plus < 1) &&
3235             ((rack->r_ctl.rc_rtt_diff <= 0) || (timely_says <= 0)))
3236                 plus = 1;
3237         if (rack->rc_gp_saw_rec &&
3238             (rack->rc_gp_no_rec_chg == 0) &&
3239             rack_bw_can_be_raised(rack, cur_bw, last_bw_est,
3240                                   rack->r_ctl.rack_per_of_gp_rec)) {
3241                 /* We have been in recovery ding it too */
3242                 calc = rack->r_ctl.rack_per_of_gp_rec + plus;
3243                 if (calc > 0xffff)
3244                         calc = 0xffff;
3245                 logged |= 1;
3246                 rack->r_ctl.rack_per_of_gp_rec = (uint16_t)calc;
3247                 if (rack_per_upper_bound_ss &&
3248                     (rack->rc_dragged_bottom == 0) &&
3249                     (rack->r_ctl.rack_per_of_gp_rec > rack_per_upper_bound_ss))
3250                         rack->r_ctl.rack_per_of_gp_rec = rack_per_upper_bound_ss;
3251         }
3252         if (rack->rc_gp_saw_ca &&
3253             (rack->rc_gp_saw_ss == 0) &&
3254             rack_bw_can_be_raised(rack, cur_bw, last_bw_est,
3255                                   rack->r_ctl.rack_per_of_gp_ca)) {
3256                 /* In CA */
3257                 calc = rack->r_ctl.rack_per_of_gp_ca + plus;
3258                 if (calc > 0xffff)
3259                         calc = 0xffff;
3260                 logged |= 2;
3261                 rack->r_ctl.rack_per_of_gp_ca = (uint16_t)calc;
3262                 if (rack_per_upper_bound_ca &&
3263                     (rack->rc_dragged_bottom == 0) &&
3264                     (rack->r_ctl.rack_per_of_gp_ca > rack_per_upper_bound_ca))
3265                         rack->r_ctl.rack_per_of_gp_ca = rack_per_upper_bound_ca;
3266         }
3267         if (rack->rc_gp_saw_ss &&
3268             rack_bw_can_be_raised(rack, cur_bw, last_bw_est,
3269                                   rack->r_ctl.rack_per_of_gp_ss)) {
3270                 /* In SS */
3271                 calc = rack->r_ctl.rack_per_of_gp_ss + plus;
3272                 if (calc > 0xffff)
3273                         calc = 0xffff;
3274                 rack->r_ctl.rack_per_of_gp_ss = (uint16_t)calc;
3275                 if (rack_per_upper_bound_ss &&
3276                     (rack->rc_dragged_bottom == 0) &&
3277                     (rack->r_ctl.rack_per_of_gp_ss > rack_per_upper_bound_ss))
3278                         rack->r_ctl.rack_per_of_gp_ss = rack_per_upper_bound_ss;
3279                 logged |= 4;
3280         }
3281         if (logged &&
3282             (rack->rc_gp_incr == 0)){
3283                 /* Go into increment mode */
3284                 rack->rc_gp_incr = 1;
3285                 rack->rc_gp_timely_inc_cnt = 0;
3286         }
3287         if (rack->rc_gp_incr &&
3288             logged &&
3289             (rack->rc_gp_timely_inc_cnt < RACK_TIMELY_CNT_BOOST)) {
3290                 rack->rc_gp_timely_inc_cnt++;
3291         }
3292         rack_log_timely(rack,  logged, plus, 0, 0,
3293                         __LINE__, 1);
3294 }
3295
3296 static uint32_t
3297 rack_get_decrease(struct tcp_rack *rack, uint32_t curper, int32_t rtt_diff)
3298 {
3299         /*
3300          * norm_grad = rtt_diff / minrtt;
3301          * new_per = curper * (1 - B * norm_grad)
3302          *
3303          * B = rack_gp_decrease_per (default 10%)
3304          * rtt_dif = input var current rtt-diff
3305          * curper = input var current percentage
3306          * minrtt = from rack filter
3307          *
3308          */
3309         uint64_t perf;
3310
3311         perf = (((uint64_t)curper * ((uint64_t)1000000 -
3312                     ((uint64_t)rack_gp_decrease_per * (uint64_t)10000 *
3313                      (((uint64_t)rtt_diff * (uint64_t)1000000)/
3314                       (uint64_t)get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt)))/
3315                      (uint64_t)1000000)) /
3316                 (uint64_t)1000000);
3317         if (perf > curper) {
3318                 /* TSNH */
3319                 perf = curper - 1;
3320         }
3321         return ((uint32_t)perf);
3322 }
3323
3324 static uint32_t
3325 rack_decrease_highrtt(struct tcp_rack *rack, uint32_t curper, uint32_t rtt)
3326 {
3327         /*
3328          *                                   highrttthresh
3329          * result = curper * (1 - (B * ( 1 -  ------          ))
3330          *                                     gp_srtt
3331          *
3332          * B = rack_gp_decrease_per (default 10%)
3333          * highrttthresh = filter_min * rack_gp_rtt_maxmul
3334          */
3335         uint64_t perf;
3336         uint32_t highrttthresh;
3337
3338         highrttthresh = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_maxmul;
3339
3340         perf = (((uint64_t)curper * ((uint64_t)1000000 -
3341                                      ((uint64_t)rack_gp_decrease_per * ((uint64_t)1000000 -
3342                                         ((uint64_t)highrttthresh * (uint64_t)1000000) /
3343                                                     (uint64_t)rtt)) / 100)) /(uint64_t)1000000);
3344         return (perf);
3345 }
3346
3347 static void
3348 rack_decrease_bw_mul(struct tcp_rack *rack, int timely_says, uint32_t rtt, int32_t rtt_diff)
3349 {
3350         uint64_t logvar, logvar2, logvar3;
3351         uint32_t logged, new_per, ss_red, ca_red, rec_red, alt, val;
3352
3353         if (rack->rc_gp_incr) {
3354                 /* Turn off increment counting */
3355                 rack->rc_gp_incr = 0;
3356                 rack->rc_gp_timely_inc_cnt = 0;
3357         }
3358         ss_red = ca_red = rec_red = 0;
3359         logged = 0;
3360         /* Calculate the reduction value */
3361         if (rtt_diff < 0) {
3362                 rtt_diff *= -1;
3363         }
3364         /* Must be at least 1% reduction */
3365         if (rack->rc_gp_saw_rec && (rack->rc_gp_no_rec_chg == 0)) {
3366                 /* We have been in recovery ding it too */
3367                 if (timely_says == 2) {
3368                         new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_rec, rtt);
3369                         alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff);
3370                         if (alt < new_per)
3371                                 val = alt;
3372                         else
3373                                 val = new_per;
3374                 } else
3375                          val = new_per = alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff);
3376                 if (rack->r_ctl.rack_per_of_gp_rec > val) {
3377                         rec_red = (rack->r_ctl.rack_per_of_gp_rec - val);
3378                         rack->r_ctl.rack_per_of_gp_rec = (uint16_t)val;
3379                 } else {
3380                         rack->r_ctl.rack_per_of_gp_rec = rack_per_lower_bound;
3381                         rec_red = 0;
3382                 }
3383                 if (rack_per_lower_bound > rack->r_ctl.rack_per_of_gp_rec)
3384                         rack->r_ctl.rack_per_of_gp_rec = rack_per_lower_bound;
3385                 logged |= 1;
3386         }
3387         if (rack->rc_gp_saw_ss) {
3388                 /* Sent in SS */
3389                 if (timely_says == 2) {
3390                         new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_ss, rtt);
3391                         alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff);
3392                         if (alt < new_per)
3393                                 val = alt;
3394                         else
3395                                 val = new_per;
3396                 } else
3397                         val = new_per = alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ss, rtt_diff);
3398                 if (rack->r_ctl.rack_per_of_gp_ss > new_per) {
3399                         ss_red = rack->r_ctl.rack_per_of_gp_ss - val;
3400                         rack->r_ctl.rack_per_of_gp_ss = (uint16_t)val;
3401                 } else {
3402                         ss_red = new_per;
3403                         rack->r_ctl.rack_per_of_gp_ss = rack_per_lower_bound;
3404                         logvar = new_per;
3405                         logvar <<= 32;
3406                         logvar |= alt;
3407                         logvar2 = (uint32_t)rtt;
3408                         logvar2 <<= 32;
3409                         logvar2 |= (uint32_t)rtt_diff;
3410                         logvar3 = rack_gp_rtt_maxmul;
3411                         logvar3 <<= 32;
3412                         logvar3 |= get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
3413                         rack_log_timely(rack, timely_says,
3414                                         logvar2, logvar3,
3415                                         logvar, __LINE__, 10);
3416                 }
3417                 if (rack_per_lower_bound > rack->r_ctl.rack_per_of_gp_ss)
3418                         rack->r_ctl.rack_per_of_gp_ss = rack_per_lower_bound;
3419                 logged |= 4;
3420         } else if (rack->rc_gp_saw_ca) {
3421                 /* Sent in CA */
3422                 if (timely_says == 2) {
3423                         new_per = rack_decrease_highrtt(rack, rack->r_ctl.rack_per_of_gp_ca, rtt);
3424                         alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_rec, rtt_diff);
3425                         if (alt < new_per)
3426                                 val = alt;
3427                         else
3428                                 val = new_per;
3429                 } else
3430                         val = new_per = alt = rack_get_decrease(rack, rack->r_ctl.rack_per_of_gp_ca, rtt_diff);
3431                 if (rack->r_ctl.rack_per_of_gp_ca > val) {
3432                         ca_red = rack->r_ctl.rack_per_of_gp_ca - val;
3433                         rack->r_ctl.rack_per_of_gp_ca = (uint16_t)val;
3434                 } else {
3435                         rack->r_ctl.rack_per_of_gp_ca = rack_per_lower_bound;
3436                         ca_red = 0;
3437                         logvar = new_per;
3438                         logvar <<= 32;
3439                         logvar |= alt;
3440                         logvar2 = (uint32_t)rtt;
3441                         logvar2 <<= 32;
3442                         logvar2 |= (uint32_t)rtt_diff;
3443                         logvar3 = rack_gp_rtt_maxmul;
3444                         logvar3 <<= 32;
3445                         logvar3 |= get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
3446                         rack_log_timely(rack, timely_says,
3447                                         logvar2, logvar3,
3448                                         logvar, __LINE__, 10);
3449                 }
3450                 if (rack_per_lower_bound > rack->r_ctl.rack_per_of_gp_ca)
3451                         rack->r_ctl.rack_per_of_gp_ca = rack_per_lower_bound;
3452                 logged |= 2;
3453         }
3454         if (rack->rc_gp_timely_dec_cnt < 0x7) {
3455                 rack->rc_gp_timely_dec_cnt++;
3456                 if (rack_timely_dec_clear &&
3457                     (rack->rc_gp_timely_dec_cnt == rack_timely_dec_clear))
3458                         rack->rc_gp_timely_dec_cnt = 0;
3459         }
3460         logvar = ss_red;
3461         logvar <<= 32;
3462         logvar |= ca_red;
3463         rack_log_timely(rack,  logged, rec_red, rack_per_lower_bound, logvar,
3464                         __LINE__, 2);
3465 }
3466
3467 static void
3468 rack_log_rtt_shrinks(struct tcp_rack *rack, uint32_t us_cts,
3469                      uint32_t rtt, uint32_t line, uint8_t reas)
3470 {
3471         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
3472                 union tcp_log_stackspecific log;
3473                 struct timeval tv;
3474
3475                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
3476                 log.u_bbr.flex1 = line;
3477                 log.u_bbr.flex2 = rack->r_ctl.rc_time_probertt_starts;
3478                 log.u_bbr.flex3 = rack->r_ctl.rc_lower_rtt_us_cts;
3479                 log.u_bbr.flex4 = rack->r_ctl.rack_per_of_gp_ss;
3480                 log.u_bbr.flex5 = rtt;
3481                 log.u_bbr.flex6 = rack->rc_highly_buffered;
3482                 log.u_bbr.flex6 <<= 1;
3483                 log.u_bbr.flex6 |= rack->forced_ack;
3484                 log.u_bbr.flex6 <<= 1;
3485                 log.u_bbr.flex6 |= rack->rc_gp_dyn_mul;
3486                 log.u_bbr.flex6 <<= 1;
3487                 log.u_bbr.flex6 |= rack->in_probe_rtt;
3488                 log.u_bbr.flex6 <<= 1;
3489                 log.u_bbr.flex6 |= rack->measure_saw_probe_rtt;
3490                 log.u_bbr.flex7 = rack->r_ctl.rack_per_of_gp_probertt;
3491                 log.u_bbr.pacing_gain = rack->r_ctl.rack_per_of_gp_ca;
3492                 log.u_bbr.cwnd_gain = rack->r_ctl.rack_per_of_gp_rec;
3493                 log.u_bbr.flex8 = reas;
3494                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
3495                 log.u_bbr.delRate = rack_get_bw(rack);
3496                 log.u_bbr.cur_del_rate = rack->r_ctl.rc_highest_us_rtt;
3497                 log.u_bbr.cur_del_rate <<= 32;
3498                 log.u_bbr.cur_del_rate |= rack->r_ctl.rc_lowest_us_rtt;
3499                 log.u_bbr.applimited = rack->r_ctl.rc_time_probertt_entered;
3500                 log.u_bbr.pkts_out = rack->r_ctl.rc_rtt_diff;
3501                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
3502                 log.u_bbr.epoch = rack->r_ctl.rc_gp_srtt;
3503                 log.u_bbr.lt_epoch = rack->r_ctl.rc_prev_gp_srtt;
3504                 log.u_bbr.pkt_epoch = rack->r_ctl.rc_lower_rtt_us_cts;
3505                 log.u_bbr.delivered = rack->r_ctl.rc_target_probertt_flight;
3506                 log.u_bbr.lost = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
3507                 log.u_bbr.rttProp = us_cts;
3508                 log.u_bbr.rttProp <<= 32;
3509                 log.u_bbr.rttProp |= rack->r_ctl.rc_entry_gp_rtt;
3510                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
3511                     &rack->rc_inp->inp_socket->so_rcv,
3512                     &rack->rc_inp->inp_socket->so_snd,
3513                     BBR_LOG_RTT_SHRINKS, 0,
3514                     0, &log, false, &rack->r_ctl.act_rcv_time);
3515         }
3516 }
3517
3518 static void
3519 rack_set_prtt_target(struct tcp_rack *rack, uint32_t segsiz, uint32_t rtt)
3520 {
3521         uint64_t bwdp;
3522
3523         bwdp = rack_get_bw(rack);
3524         bwdp *= (uint64_t)rtt;
3525         bwdp /= (uint64_t)HPTS_USEC_IN_SEC;
3526         rack->r_ctl.rc_target_probertt_flight = roundup((uint32_t)bwdp, segsiz);
3527         if (rack->r_ctl.rc_target_probertt_flight < (segsiz * rack_timely_min_segs)) {
3528                 /*
3529                  * A window protocol must be able to have 4 packets
3530                  * outstanding as the floor in order to function
3531                  * (especially considering delayed ack :D).
3532                  */
3533                 rack->r_ctl.rc_target_probertt_flight = (segsiz * rack_timely_min_segs);
3534         }
3535 }
3536
3537 static void
3538 rack_enter_probertt(struct tcp_rack *rack, uint32_t us_cts)
3539 {
3540         /**
3541          * ProbeRTT is a bit different in rack_pacing than in
3542          * BBR. It is like BBR in that it uses the lowering of
3543          * the RTT as a signal that we saw something new and
3544          * counts from there for how long between. But it is
3545          * different in that its quite simple. It does not
3546          * play with the cwnd and wait until we get down
3547          * to N segments outstanding and hold that for
3548          * 200ms. Instead it just sets the pacing reduction
3549          * rate to a set percentage (70 by default) and hold
3550          * that for a number of recent GP Srtt's.
3551          */
3552         uint32_t segsiz;
3553
3554         if (rack->rc_gp_dyn_mul == 0)
3555                 return;
3556
3557         if (rack->rc_tp->snd_max == rack->rc_tp->snd_una) {
3558                 /* We are idle */
3559                 return;
3560         }
3561         if ((rack->rc_tp->t_flags & TF_GPUTINPROG) &&
3562             SEQ_GT(rack->rc_tp->snd_una, rack->rc_tp->gput_seq)) {
3563                 /*
3564                  * Stop the goodput now, the idea here is
3565                  * that future measurements with in_probe_rtt
3566                  * won't register if they are not greater so
3567                  * we want to get what info (if any) is available
3568                  * now.
3569                  */
3570                 rack_do_goodput_measurement(rack->rc_tp, rack,
3571                                             rack->rc_tp->snd_una, __LINE__,
3572                                             RACK_QUALITY_PROBERTT);
3573         }
3574         rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt;
3575         rack->r_ctl.rc_time_probertt_entered = us_cts;
3576         segsiz = min(ctf_fixed_maxseg(rack->rc_tp),
3577                      rack->r_ctl.rc_pace_min_segs);
3578         rack->in_probe_rtt = 1;
3579         rack->measure_saw_probe_rtt = 1;
3580         rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
3581         rack->r_ctl.rc_time_probertt_starts = 0;
3582         rack->r_ctl.rc_entry_gp_rtt = rack->r_ctl.rc_gp_srtt;
3583         if (rack_probertt_use_min_rtt_entry)
3584                 rack_set_prtt_target(rack, segsiz, get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt));
3585         else
3586                 rack_set_prtt_target(rack, segsiz, rack->r_ctl.rc_gp_srtt);
3587         rack_log_rtt_shrinks(rack,  us_cts,  get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
3588                              __LINE__, RACK_RTTS_ENTERPROBE);
3589 }
3590
3591 static void
3592 rack_exit_probertt(struct tcp_rack *rack, uint32_t us_cts)
3593 {
3594         struct rack_sendmap *rsm;
3595         uint32_t segsiz;
3596
3597         segsiz = min(ctf_fixed_maxseg(rack->rc_tp),
3598                      rack->r_ctl.rc_pace_min_segs);
3599         rack->in_probe_rtt = 0;
3600         if ((rack->rc_tp->t_flags & TF_GPUTINPROG) &&
3601             SEQ_GT(rack->rc_tp->snd_una, rack->rc_tp->gput_seq)) {
3602                 /*
3603                  * Stop the goodput now, the idea here is
3604                  * that future measurements with in_probe_rtt
3605                  * won't register if they are not greater so
3606                  * we want to get what info (if any) is available
3607                  * now.
3608                  */
3609                 rack_do_goodput_measurement(rack->rc_tp, rack,
3610                                             rack->rc_tp->snd_una, __LINE__,
3611                                             RACK_QUALITY_PROBERTT);
3612         } else if (rack->rc_tp->t_flags & TF_GPUTINPROG) {
3613                 /*
3614                  * We don't have enough data to make a measurement.
3615                  * So lets just stop and start here after exiting
3616                  * probe-rtt. We probably are not interested in
3617                  * the results anyway.
3618                  */
3619                 rack->rc_tp->t_flags &= ~TF_GPUTINPROG;
3620         }
3621         /*
3622          * Measurements through the current snd_max are going
3623          * to be limited by the slower pacing rate.
3624          *
3625          * We need to mark these as app-limited so we
3626          * don't collapse the b/w.
3627          */
3628         rsm = RB_MAX(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
3629         if (rsm && ((rsm->r_flags & RACK_APP_LIMITED) == 0)) {
3630                 if (rack->r_ctl.rc_app_limited_cnt == 0)
3631                         rack->r_ctl.rc_end_appl = rack->r_ctl.rc_first_appl = rsm;
3632                 else {
3633                         /*
3634                          * Go out to the end app limited and mark
3635                          * this new one as next and move the end_appl up
3636                          * to this guy.
3637                          */
3638                         if (rack->r_ctl.rc_end_appl)
3639                                 rack->r_ctl.rc_end_appl->r_nseq_appl = rsm->r_start;
3640                         rack->r_ctl.rc_end_appl = rsm;
3641                 }
3642                 rsm->r_flags |= RACK_APP_LIMITED;
3643                 rack->r_ctl.rc_app_limited_cnt++;
3644         }
3645         /*
3646          * Now, we need to examine our pacing rate multipliers.
3647          * If its under 100%, we need to kick it back up to
3648          * 100%. We also don't let it be over our "max" above
3649          * the actual rate i.e. 100% + rack_clamp_atexit_prtt.
3650          * Note setting clamp_atexit_prtt to 0 has the effect
3651          * of setting CA/SS to 100% always at exit (which is
3652          * the default behavior).
3653          */
3654         if (rack_probertt_clear_is) {
3655                 rack->rc_gp_incr = 0;
3656                 rack->rc_gp_bwred = 0;
3657                 rack->rc_gp_timely_inc_cnt = 0;
3658                 rack->rc_gp_timely_dec_cnt = 0;
3659         }
3660         /* Do we do any clamping at exit? */
3661         if (rack->rc_highly_buffered && rack_atexit_prtt_hbp) {
3662                 rack->r_ctl.rack_per_of_gp_ca = rack_atexit_prtt_hbp;
3663                 rack->r_ctl.rack_per_of_gp_ss = rack_atexit_prtt_hbp;
3664         }
3665         if ((rack->rc_highly_buffered == 0) && rack_atexit_prtt) {
3666                 rack->r_ctl.rack_per_of_gp_ca = rack_atexit_prtt;
3667                 rack->r_ctl.rack_per_of_gp_ss = rack_atexit_prtt;
3668         }
3669         /*
3670          * Lets set rtt_diff to 0, so that we will get a "boost"
3671          * after exiting.
3672          */
3673         rack->r_ctl.rc_rtt_diff = 0;
3674
3675         /* Clear all flags so we start fresh */
3676         rack->rc_tp->t_bytes_acked = 0;
3677         rack->rc_tp->ccv->flags &= ~CCF_ABC_SENTAWND;
3678         /*
3679          * If configured to, set the cwnd and ssthresh to
3680          * our targets.
3681          */
3682         if (rack_probe_rtt_sets_cwnd) {
3683                 uint64_t ebdp;
3684                 uint32_t setto;
3685
3686                 /* Set ssthresh so we get into CA once we hit our target */
3687                 if (rack_probertt_use_min_rtt_exit == 1) {
3688                         /* Set to min rtt */
3689                         rack_set_prtt_target(rack, segsiz,
3690                                              get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt));
3691                 } else if (rack_probertt_use_min_rtt_exit == 2) {
3692                         /* Set to current gp rtt */
3693                         rack_set_prtt_target(rack, segsiz,
3694                                              rack->r_ctl.rc_gp_srtt);
3695                 } else if (rack_probertt_use_min_rtt_exit == 3) {
3696                         /* Set to entry gp rtt */
3697                         rack_set_prtt_target(rack, segsiz,
3698                                              rack->r_ctl.rc_entry_gp_rtt);
3699                 } else {
3700                         uint64_t sum;
3701                         uint32_t setval;
3702
3703                         sum = rack->r_ctl.rc_entry_gp_rtt;
3704                         sum *= 10;
3705                         sum /= (uint64_t)(max(1, rack->r_ctl.rc_gp_srtt));
3706                         if (sum >= 20) {
3707                                 /*
3708                                  * A highly buffered path needs
3709                                  * cwnd space for timely to work.
3710                                  * Lets set things up as if
3711                                  * we are heading back here again.
3712                                  */
3713                                 setval = rack->r_ctl.rc_entry_gp_rtt;
3714                         } else if (sum >= 15) {
3715                                 /*
3716                                  * Lets take the smaller of the
3717                                  * two since we are just somewhat
3718                                  * buffered.
3719                                  */
3720                                 setval = rack->r_ctl.rc_gp_srtt;
3721                                 if (setval > rack->r_ctl.rc_entry_gp_rtt)
3722                                         setval = rack->r_ctl.rc_entry_gp_rtt;
3723                         } else {
3724                                 /*
3725                                  * Here we are not highly buffered
3726                                  * and should pick the min we can to
3727                                  * keep from causing loss.
3728                                  */
3729                                 setval = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
3730                         }
3731                         rack_set_prtt_target(rack, segsiz,
3732                                              setval);
3733                 }
3734                 if (rack_probe_rtt_sets_cwnd > 1) {
3735                         /* There is a percentage here to boost */
3736                         ebdp = rack->r_ctl.rc_target_probertt_flight;
3737                         ebdp *= rack_probe_rtt_sets_cwnd;
3738                         ebdp /= 100;
3739                         setto = rack->r_ctl.rc_target_probertt_flight + ebdp;
3740                 } else
3741                         setto = rack->r_ctl.rc_target_probertt_flight;
3742                 rack->rc_tp->snd_cwnd = roundup(setto, segsiz);
3743                 if (rack->rc_tp->snd_cwnd < (segsiz * rack_timely_min_segs)) {
3744                         /* Enforce a min */
3745                         rack->rc_tp->snd_cwnd = segsiz * rack_timely_min_segs;
3746                 }
3747                 /* If we set in the cwnd also set the ssthresh point so we are in CA */
3748                 rack->rc_tp->snd_ssthresh = (rack->rc_tp->snd_cwnd - 1);
3749         }
3750         rack_log_rtt_shrinks(rack,  us_cts,
3751                              get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
3752                              __LINE__, RACK_RTTS_EXITPROBE);
3753         /* Clear times last so log has all the info */
3754         rack->r_ctl.rc_probertt_sndmax_atexit = rack->rc_tp->snd_max;
3755         rack->r_ctl.rc_time_probertt_entered = us_cts;
3756         rack->r_ctl.rc_time_probertt_starts = rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
3757         rack->r_ctl.rc_time_of_last_probertt = us_cts;
3758 }
3759
3760 static void
3761 rack_check_probe_rtt(struct tcp_rack *rack, uint32_t us_cts)
3762 {
3763         /* Check in on probe-rtt */
3764         if (rack->rc_gp_filled == 0) {
3765                 /* We do not do p-rtt unless we have gp measurements */
3766                 return;
3767         }
3768         if (rack->in_probe_rtt) {
3769                 uint64_t no_overflow;
3770                 uint32_t endtime, must_stay;
3771
3772                 if (rack->r_ctl.rc_went_idle_time &&
3773                     ((us_cts - rack->r_ctl.rc_went_idle_time) > rack_min_probertt_hold)) {
3774                         /*
3775                          * We went idle during prtt, just exit now.
3776                          */
3777                         rack_exit_probertt(rack, us_cts);
3778                 } else if (rack_probe_rtt_safety_val &&
3779                     TSTMP_GT(us_cts, rack->r_ctl.rc_time_probertt_entered) &&
3780                     ((us_cts - rack->r_ctl.rc_time_probertt_entered) > rack_probe_rtt_safety_val)) {
3781                         /*
3782                          * Probe RTT safety value triggered!
3783                          */
3784                         rack_log_rtt_shrinks(rack,  us_cts,
3785                                              get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
3786                                              __LINE__, RACK_RTTS_SAFETY);
3787                         rack_exit_probertt(rack, us_cts);
3788                 }
3789                 /* Calculate the max we will wait */
3790                 endtime = rack->r_ctl.rc_time_probertt_entered + (rack->r_ctl.rc_gp_srtt * rack_max_drain_wait);
3791                 if (rack->rc_highly_buffered)
3792                         endtime += (rack->r_ctl.rc_gp_srtt * rack_max_drain_hbp);
3793                 /* Calculate the min we must wait */
3794                 must_stay = rack->r_ctl.rc_time_probertt_entered + (rack->r_ctl.rc_gp_srtt * rack_must_drain);
3795                 if ((ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) > rack->r_ctl.rc_target_probertt_flight) &&
3796                     TSTMP_LT(us_cts, endtime)) {
3797                         uint32_t calc;
3798                         /* Do we lower more? */
3799 no_exit:
3800                         if (TSTMP_GT(us_cts, rack->r_ctl.rc_time_probertt_entered))
3801                                 calc = us_cts - rack->r_ctl.rc_time_probertt_entered;
3802                         else
3803                                 calc = 0;
3804                         calc /= max(rack->r_ctl.rc_gp_srtt, 1);
3805                         if (calc) {
3806                                 /* Maybe */
3807                                 calc *= rack_per_of_gp_probertt_reduce;
3808                                 rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt - calc;
3809                                 /* Limit it too */
3810                                 if (rack->r_ctl.rack_per_of_gp_probertt < rack_per_of_gp_lowthresh)
3811                                         rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_lowthresh;
3812                         }
3813                         /* We must reach target or the time set */
3814                         return;
3815                 }
3816                 if (rack->r_ctl.rc_time_probertt_starts == 0) {
3817                         if ((TSTMP_LT(us_cts, must_stay) &&
3818                              rack->rc_highly_buffered) ||
3819                              (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) >
3820                               rack->r_ctl.rc_target_probertt_flight)) {
3821                                 /* We are not past the must_stay time */
3822                                 goto no_exit;
3823                         }
3824                         rack_log_rtt_shrinks(rack,  us_cts,
3825                                              get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
3826                                              __LINE__, RACK_RTTS_REACHTARGET);
3827                         rack->r_ctl.rc_time_probertt_starts = us_cts;
3828                         if (rack->r_ctl.rc_time_probertt_starts == 0)
3829                                 rack->r_ctl.rc_time_probertt_starts = 1;
3830                         /* Restore back to our rate we want to pace at in prtt */
3831                         rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt;
3832                 }
3833                 /*
3834                  * Setup our end time, some number of gp_srtts plus 200ms.
3835                  */
3836                 no_overflow = ((uint64_t)rack->r_ctl.rc_gp_srtt *
3837                                (uint64_t)rack_probertt_gpsrtt_cnt_mul);
3838                 if (rack_probertt_gpsrtt_cnt_div)
3839                         endtime = (uint32_t)(no_overflow / (uint64_t)rack_probertt_gpsrtt_cnt_div);
3840                 else
3841                         endtime = 0;
3842                 endtime += rack_min_probertt_hold;
3843                 endtime += rack->r_ctl.rc_time_probertt_starts;
3844                 if (TSTMP_GEQ(us_cts,  endtime)) {
3845                         /* yes, exit probertt */
3846                         rack_exit_probertt(rack, us_cts);
3847                 }
3848
3849         } else if ((us_cts - rack->r_ctl.rc_lower_rtt_us_cts) >= rack_time_between_probertt) {
3850                 /* Go into probertt, its been too long since we went lower */
3851                 rack_enter_probertt(rack, us_cts);
3852         }
3853 }
3854
3855 static void
3856 rack_update_multiplier(struct tcp_rack *rack, int32_t timely_says, uint64_t last_bw_est,
3857                        uint32_t rtt, int32_t rtt_diff)
3858 {
3859         uint64_t cur_bw, up_bnd, low_bnd, subfr;
3860         uint32_t losses;
3861
3862         if ((rack->rc_gp_dyn_mul == 0) ||
3863             (rack->use_fixed_rate) ||
3864             (rack->in_probe_rtt) ||
3865             (rack->rc_always_pace == 0)) {
3866                 /* No dynamic GP multiplier in play */
3867                 return;
3868         }
3869         losses = rack->r_ctl.rc_loss_count - rack->r_ctl.rc_loss_at_start;
3870         cur_bw = rack_get_bw(rack);
3871         /* Calculate our up and down range */
3872         up_bnd = rack->r_ctl.last_gp_comp_bw * (uint64_t)rack_gp_per_bw_mul_up;
3873         up_bnd /= 100;
3874         up_bnd += rack->r_ctl.last_gp_comp_bw;
3875
3876         subfr = (uint64_t)rack->r_ctl.last_gp_comp_bw * (uint64_t)rack_gp_per_bw_mul_down;
3877         subfr /= 100;
3878         low_bnd = rack->r_ctl.last_gp_comp_bw - subfr;
3879         if ((timely_says == 2) && (rack->r_ctl.rc_no_push_at_mrtt)) {
3880                 /*
3881                  * This is the case where our RTT is above
3882                  * the max target and we have been configured
3883                  * to just do timely no bonus up stuff in that case.
3884                  *
3885                  * There are two configurations, set to 1, and we
3886                  * just do timely if we are over our max. If its
3887                  * set above 1 then we slam the multipliers down
3888                  * to 100 and then decrement per timely.
3889                  */
3890                 rack_log_timely(rack,  timely_says, cur_bw, low_bnd, up_bnd,
3891                                 __LINE__, 3);
3892                 if (rack->r_ctl.rc_no_push_at_mrtt > 1)
3893                         rack_validate_multipliers_at_or_below_100(rack);
3894                 rack_decrease_bw_mul(rack, timely_says, rtt, rtt_diff);
3895         } else if ((last_bw_est < low_bnd) && !losses) {
3896                 /*
3897                  * We are decreasing this is a bit complicated this
3898                  * means we are loosing ground. This could be
3899                  * because another flow entered and we are competing
3900                  * for b/w with it. This will push the RTT up which
3901                  * makes timely unusable unless we want to get shoved
3902                  * into a corner and just be backed off (the age
3903                  * old problem with delay based CC).
3904                  *
3905                  * On the other hand if it was a route change we
3906                  * would like to stay somewhat contained and not
3907                  * blow out the buffers.
3908                  */
3909                 rack_log_timely(rack,  timely_says, cur_bw, low_bnd, up_bnd,
3910                                 __LINE__, 3);
3911                 rack->r_ctl.last_gp_comp_bw = cur_bw;
3912                 if (rack->rc_gp_bwred == 0) {
3913                         /* Go into reduction counting */
3914                         rack->rc_gp_bwred = 1;
3915                         rack->rc_gp_timely_dec_cnt = 0;
3916                 }
3917                 if ((rack->rc_gp_timely_dec_cnt < rack_timely_max_push_drop) ||
3918                     (timely_says == 0)) {
3919                         /*
3920                          * Push another time with a faster pacing
3921                          * to try to gain back (we include override to
3922                          * get a full raise factor).
3923                          */
3924                         if ((rack->rc_gp_saw_ca && rack->r_ctl.rack_per_of_gp_ca <= rack_down_raise_thresh) ||
3925                             (rack->rc_gp_saw_ss && rack->r_ctl.rack_per_of_gp_ss <= rack_down_raise_thresh) ||
3926                             (timely_says == 0) ||
3927                             (rack_down_raise_thresh == 0)) {
3928                                 /*
3929                                  * Do an override up in b/w if we were
3930                                  * below the threshold or if the threshold
3931                                  * is zero we always do the raise.
3932                                  */
3933                                 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 1);
3934                         } else {
3935                                 /* Log it stays the same */
3936                                 rack_log_timely(rack,  0, last_bw_est, low_bnd, 0,
3937                                                 __LINE__, 11);
3938                         }
3939                         rack->rc_gp_timely_dec_cnt++;
3940                         /* We are not incrementing really no-count */
3941                         rack->rc_gp_incr = 0;
3942                         rack->rc_gp_timely_inc_cnt = 0;
3943                 } else {
3944                         /*
3945                          * Lets just use the RTT
3946                          * information and give up
3947                          * pushing.
3948                          */
3949                         goto use_timely;
3950                 }
3951         } else if ((timely_says != 2) &&
3952                     !losses &&
3953                     (last_bw_est > up_bnd)) {
3954                 /*
3955                  * We are increasing b/w lets keep going, updating
3956                  * our b/w and ignoring any timely input, unless
3957                  * of course we are at our max raise (if there is one).
3958                  */
3959
3960                 rack_log_timely(rack,  timely_says, cur_bw, low_bnd, up_bnd,
3961                                 __LINE__, 3);
3962                 rack->r_ctl.last_gp_comp_bw = cur_bw;
3963                 if (rack->rc_gp_saw_ss &&
3964                     rack_per_upper_bound_ss &&
3965                      (rack->r_ctl.rack_per_of_gp_ss == rack_per_upper_bound_ss)) {
3966                             /*
3967                              * In cases where we can't go higher
3968                              * we should just use timely.
3969                              */
3970                             goto use_timely;
3971                 }
3972                 if (rack->rc_gp_saw_ca &&
3973                     rack_per_upper_bound_ca &&
3974                     (rack->r_ctl.rack_per_of_gp_ca == rack_per_upper_bound_ca)) {
3975                             /*
3976                              * In cases where we can't go higher
3977                              * we should just use timely.
3978                              */
3979                             goto use_timely;
3980                 }
3981                 rack->rc_gp_bwred = 0;
3982                 rack->rc_gp_timely_dec_cnt = 0;
3983                 /* You get a set number of pushes if timely is trying to reduce */
3984                 if ((rack->rc_gp_incr < rack_timely_max_push_rise) || (timely_says == 0)) {
3985                         rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 0);
3986                 } else {
3987                         /* Log it stays the same */
3988                         rack_log_timely(rack,  0, last_bw_est, up_bnd, 0,
3989                             __LINE__, 12);
3990                 }
3991                 return;
3992         } else {
3993                 /*
3994                  * We are staying between the lower and upper range bounds
3995                  * so use timely to decide.
3996                  */
3997                 rack_log_timely(rack,  timely_says, cur_bw, low_bnd, up_bnd,
3998                                 __LINE__, 3);
3999 use_timely:
4000                 if (timely_says) {
4001                         rack->rc_gp_incr = 0;
4002                         rack->rc_gp_timely_inc_cnt = 0;
4003                         if ((rack->rc_gp_timely_dec_cnt < rack_timely_max_push_drop) &&
4004                             !losses &&
4005                             (last_bw_est < low_bnd)) {
4006                                 /* We are loosing ground */
4007                                 rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 0);
4008                                 rack->rc_gp_timely_dec_cnt++;
4009                                 /* We are not incrementing really no-count */
4010                                 rack->rc_gp_incr = 0;
4011                                 rack->rc_gp_timely_inc_cnt = 0;
4012                         } else
4013                                 rack_decrease_bw_mul(rack, timely_says, rtt, rtt_diff);
4014                 } else {
4015                         rack->rc_gp_bwred = 0;
4016                         rack->rc_gp_timely_dec_cnt = 0;
4017                         rack_increase_bw_mul(rack, timely_says, cur_bw, last_bw_est, 0);
4018                 }
4019         }
4020 }
4021
4022 static int32_t
4023 rack_make_timely_judgement(struct tcp_rack *rack, uint32_t rtt, int32_t rtt_diff, uint32_t prev_rtt)
4024 {
4025         int32_t timely_says;
4026         uint64_t log_mult, log_rtt_a_diff;
4027
4028         log_rtt_a_diff = rtt;
4029         log_rtt_a_diff <<= 32;
4030         log_rtt_a_diff |= (uint32_t)rtt_diff;
4031         if (rtt >= (get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) *
4032                     rack_gp_rtt_maxmul)) {
4033                 /* Reduce the b/w multiplier */
4034                 timely_says = 2;
4035                 log_mult = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_maxmul;
4036                 log_mult <<= 32;
4037                 log_mult |= prev_rtt;
4038                 rack_log_timely(rack,  timely_says, log_mult,
4039                                 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
4040                                 log_rtt_a_diff, __LINE__, 4);
4041         } else if (rtt <= (get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) +
4042                            ((get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_minmul) /
4043                             max(rack_gp_rtt_mindiv , 1)))) {
4044                 /* Increase the b/w multiplier */
4045                 log_mult = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) +
4046                         ((get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack_gp_rtt_minmul) /
4047                          max(rack_gp_rtt_mindiv , 1));
4048                 log_mult <<= 32;
4049                 log_mult |= prev_rtt;
4050                 timely_says = 0;
4051                 rack_log_timely(rack,  timely_says, log_mult ,
4052                                 get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt),
4053                                 log_rtt_a_diff, __LINE__, 5);
4054         } else {
4055                 /*
4056                  * Use a gradient to find it the timely gradient
4057                  * is:
4058                  * grad = rc_rtt_diff / min_rtt;
4059                  *
4060                  * anything below or equal to 0 will be
4061                  * a increase indication. Anything above
4062                  * zero is a decrease. Note we take care
4063                  * of the actual gradient calculation
4064                  * in the reduction (its not needed for
4065                  * increase).
4066                  */
4067                 log_mult = prev_rtt;
4068                 if (rtt_diff <= 0) {
4069                         /*
4070                          * Rttdiff is less than zero, increase the
4071                          * b/w multiplier (its 0 or negative)
4072                          */
4073                         timely_says = 0;
4074                         rack_log_timely(rack,  timely_says, log_mult,
4075                                         get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), log_rtt_a_diff, __LINE__, 6);
4076                 } else {
4077                         /* Reduce the b/w multiplier */
4078                         timely_says = 1;
4079                         rack_log_timely(rack,  timely_says, log_mult,
4080                                         get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt), log_rtt_a_diff, __LINE__, 7);
4081                 }
4082         }
4083         return (timely_says);
4084 }
4085
4086 static void
4087 rack_do_goodput_measurement(struct tcpcb *tp, struct tcp_rack *rack,
4088                             tcp_seq th_ack, int line, uint8_t quality)
4089 {
4090         uint64_t tim, bytes_ps, ltim, stim, utim;
4091         uint32_t segsiz, bytes, reqbytes, us_cts;
4092         int32_t gput, new_rtt_diff, timely_says;
4093         uint64_t  resid_bw, subpart = 0, addpart = 0, srtt;
4094         int did_add = 0;
4095
4096         us_cts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time);
4097         segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
4098         if (TSTMP_GEQ(us_cts, tp->gput_ts))
4099                 tim = us_cts - tp->gput_ts;
4100         else
4101                 tim = 0;
4102         if (rack->r_ctl.rc_gp_cumack_ts > rack->r_ctl.rc_gp_output_ts)
4103                 stim = rack->r_ctl.rc_gp_cumack_ts - rack->r_ctl.rc_gp_output_ts;
4104         else
4105                 stim = 0;
4106         /*
4107          * Use the larger of the send time or ack time. This prevents us
4108          * from being influenced by ack artifacts to come up with too
4109          * high of measurement. Note that since we are spanning over many more
4110          * bytes in most of our measurements hopefully that is less likely to
4111          * occur.
4112          */
4113         if (tim > stim)
4114                 utim = max(tim, 1);
4115         else
4116                 utim = max(stim, 1);
4117         /* Lets get a msec time ltim too for the old stuff */
4118         ltim = max(1, (utim / HPTS_USEC_IN_MSEC));
4119         gput = (((uint64_t) (th_ack - tp->gput_seq)) << 3) / ltim;
4120         reqbytes = min(rc_init_window(rack), (MIN_GP_WIN * segsiz));
4121         if ((tim == 0) && (stim == 0)) {
4122                 /*
4123                  * Invalid measurement time, maybe
4124                  * all on one ack/one send?
4125                  */
4126                 bytes = 0;
4127                 bytes_ps = 0;
4128                 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
4129                                            0, 0, 0, 10, __LINE__, NULL, quality);
4130                 goto skip_measurement;
4131         }
4132         if (rack->r_ctl.rc_gp_lowrtt == 0xffffffff) {
4133                 /* We never made a us_rtt measurement? */
4134                 bytes = 0;
4135                 bytes_ps = 0;
4136                 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
4137                                            0, 0, 0, 10, __LINE__, NULL, quality);
4138                 goto skip_measurement;
4139         }
4140         /*
4141          * Calculate the maximum possible b/w this connection
4142          * could have. We base our calculation on the lowest
4143          * rtt we have seen during the measurement and the
4144          * largest rwnd the client has given us in that time. This
4145          * forms a BDP that is the maximum that we could ever
4146          * get to the client. Anything larger is not valid.
4147          *
4148          * I originally had code here that rejected measurements
4149          * where the time was less than 1/2 the latest us_rtt.
4150          * But after thinking on that I realized its wrong since
4151          * say you had a 150Mbps or even 1Gbps link, and you
4152          * were a long way away.. example I am in Europe (100ms rtt)
4153          * talking to my 1Gbps link in S.C. Now measuring say 150,000
4154          * bytes my time would be 1.2ms, and yet my rtt would say
4155          * the measurement was invalid the time was < 50ms. The
4156          * same thing is true for 150Mb (8ms of time).
4157          *
4158          * A better way I realized is to look at what the maximum
4159          * the connection could possibly do. This is gated on
4160          * the lowest RTT we have seen and the highest rwnd.
4161          * We should in theory never exceed that, if we are
4162          * then something on the path is storing up packets
4163          * and then feeding them all at once to our endpoint
4164          * messing up our measurement.
4165          */
4166         rack->r_ctl.last_max_bw = rack->r_ctl.rc_gp_high_rwnd;
4167         rack->r_ctl.last_max_bw *= HPTS_USEC_IN_SEC;
4168         rack->r_ctl.last_max_bw /= rack->r_ctl.rc_gp_lowrtt;
4169         if (SEQ_LT(th_ack, tp->gput_seq)) {
4170                 /* No measurement can be made */
4171                 bytes = 0;
4172                 bytes_ps = 0;
4173                 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
4174                                            0, 0, 0, 10, __LINE__, NULL, quality);
4175                 goto skip_measurement;
4176         } else
4177                 bytes = (th_ack - tp->gput_seq);
4178         bytes_ps = (uint64_t)bytes;
4179         /*
4180          * Don't measure a b/w for pacing unless we have gotten at least
4181          * an initial windows worth of data in this measurement interval.
4182          *
4183          * Small numbers of bytes get badly influenced by delayed ack and
4184          * other artifacts. Note we take the initial window or our
4185          * defined minimum GP (defaulting to 10 which hopefully is the
4186          * IW).
4187          */
4188         if (rack->rc_gp_filled == 0) {
4189                 /*
4190                  * The initial estimate is special. We
4191                  * have blasted out an IW worth of packets
4192                  * without a real valid ack ts results. We
4193                  * then setup the app_limited_needs_set flag,
4194                  * this should get the first ack in (probably 2
4195                  * MSS worth) to be recorded as the timestamp.
4196                  * We thus allow a smaller number of bytes i.e.
4197                  * IW - 2MSS.
4198                  */
4199                 reqbytes -= (2 * segsiz);
4200                 /* Also lets fill previous for our first measurement to be neutral */
4201                 rack->r_ctl.rc_prev_gp_srtt = rack->r_ctl.rc_gp_srtt;
4202         }
4203         if ((bytes_ps < reqbytes) || rack->app_limited_needs_set) {
4204                 rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
4205                                            rack->r_ctl.rc_app_limited_cnt,
4206                                            0, 0, 10, __LINE__, NULL, quality);
4207                 goto skip_measurement;
4208         }
4209         /*
4210          * We now need to calculate the Timely like status so
4211          * we can update (possibly) the b/w multipliers.
4212          */
4213         new_rtt_diff = (int32_t)rack->r_ctl.rc_gp_srtt - (int32_t)rack->r_ctl.rc_prev_gp_srtt;
4214         if (rack->rc_gp_filled == 0) {
4215                 /* No previous reading */
4216                 rack->r_ctl.rc_rtt_diff = new_rtt_diff;
4217         } else {
4218                 if (rack->measure_saw_probe_rtt == 0) {
4219                         /*
4220                          * We don't want a probertt to be counted
4221                          * since it will be negative incorrectly. We
4222                          * expect to be reducing the RTT when we
4223                          * pace at a slower rate.
4224                          */
4225                         rack->r_ctl.rc_rtt_diff -= (rack->r_ctl.rc_rtt_diff / 8);
4226                         rack->r_ctl.rc_rtt_diff += (new_rtt_diff / 8);
4227                 }
4228         }
4229         timely_says = rack_make_timely_judgement(rack,
4230                 rack->r_ctl.rc_gp_srtt,
4231                 rack->r_ctl.rc_rtt_diff,
4232                 rack->r_ctl.rc_prev_gp_srtt
4233                 );
4234         bytes_ps *= HPTS_USEC_IN_SEC;
4235         bytes_ps /= utim;
4236         if (bytes_ps > rack->r_ctl.last_max_bw) {
4237                 /*
4238                  * Something is on path playing
4239                  * since this b/w is not possible based
4240                  * on our BDP (highest rwnd and lowest rtt
4241                  * we saw in the measurement window).
4242                  *
4243                  * Another option here would be to
4244                  * instead skip the measurement.
4245                  */
4246                 rack_log_pacing_delay_calc(rack, bytes, reqbytes,
4247                                            bytes_ps, rack->r_ctl.last_max_bw, 0,
4248                                            11, __LINE__, NULL, quality);
4249                 bytes_ps = rack->r_ctl.last_max_bw;
4250         }
4251         /* We store gp for b/w in bytes per second */
4252         if (rack->rc_gp_filled == 0) {
4253                 /* Initial measurement */
4254                 if (bytes_ps) {
4255                         rack->r_ctl.gp_bw = bytes_ps;
4256                         rack->rc_gp_filled = 1;
4257                         rack->r_ctl.num_measurements = 1;
4258                         rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
4259                 } else {
4260                         rack_log_pacing_delay_calc(rack, bytes_ps, reqbytes,
4261                                                    rack->r_ctl.rc_app_limited_cnt,
4262                                                    0, 0, 10, __LINE__, NULL, quality);
4263                 }
4264                 if (tcp_in_hpts(rack->rc_inp) &&
4265                     (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
4266                         /*
4267                          * Ok we can't trust the pacer in this case
4268                          * where we transition from un-paced to paced.
4269                          * Or for that matter when the burst mitigation
4270                          * was making a wild guess and got it wrong.
4271                          * Stop the pacer and clear up all the aggregate
4272                          * delays etc.
4273                          */
4274                         tcp_hpts_remove(rack->rc_inp);
4275                         rack->r_ctl.rc_hpts_flags = 0;
4276                         rack->r_ctl.rc_last_output_to = 0;
4277                 }
4278                 did_add = 2;
4279         } else if (rack->r_ctl.num_measurements < RACK_REQ_AVG) {
4280                 /* Still a small number run an average */
4281                 rack->r_ctl.gp_bw += bytes_ps;
4282                 addpart = rack->r_ctl.num_measurements;
4283                 rack->r_ctl.num_measurements++;
4284                 if (rack->r_ctl.num_measurements >= RACK_REQ_AVG) {
4285                         /* We have collected enough to move forward */
4286                         rack->r_ctl.gp_bw /= (uint64_t)rack->r_ctl.num_measurements;
4287                 }
4288                 did_add = 3;
4289         } else {
4290                 /*
4291                  * We want to take 1/wma of the goodput and add in to 7/8th
4292                  * of the old value weighted by the srtt. So if your measurement
4293                  * period is say 2 SRTT's long you would get 1/4 as the
4294                  * value, if it was like 1/2 SRTT then you would get 1/16th.
4295                  *
4296                  * But we must be careful not to take too much i.e. if the
4297                  * srtt is say 20ms and the measurement is taken over
4298                  * 400ms our weight would be 400/20 i.e. 20. On the
4299                  * other hand if we get a measurement over 1ms with a
4300                  * 10ms rtt we only want to take a much smaller portion.
4301                  */
4302                 if (rack->r_ctl.num_measurements < 0xff) {
4303                         rack->r_ctl.num_measurements++;
4304                 }
4305                 srtt = (uint64_t)tp->t_srtt;
4306                 if (srtt == 0) {
4307                         /*
4308                          * Strange why did t_srtt go back to zero?
4309                          */
4310                         if (rack->r_ctl.rc_rack_min_rtt)
4311                                 srtt = rack->r_ctl.rc_rack_min_rtt;
4312                         else
4313                                 srtt = HPTS_USEC_IN_MSEC;
4314                 }
4315                 /*
4316                  * XXXrrs: Note for reviewers, in playing with
4317                  * dynamic pacing I discovered this GP calculation
4318                  * as done originally leads to some undesired results.
4319                  * Basically you can get longer measurements contributing
4320                  * too much to the WMA. Thus I changed it if you are doing
4321                  * dynamic adjustments to only do the aportioned adjustment
4322                  * if we have a very small (time wise) measurement. Longer
4323                  * measurements just get there weight (defaulting to 1/8)
4324                  * add to the WMA. We may want to think about changing
4325                  * this to always do that for both sides i.e. dynamic
4326                  * and non-dynamic... but considering lots of folks
4327                  * were playing with this I did not want to change the
4328                  * calculation per.se. without your thoughts.. Lawerence?
4329                  * Peter??
4330                  */
4331                 if (rack->rc_gp_dyn_mul == 0) {
4332                         subpart = rack->r_ctl.gp_bw * utim;
4333                         subpart /= (srtt * 8);
4334                         if (subpart < (rack->r_ctl.gp_bw / 2)) {
4335                                 /*
4336                                  * The b/w update takes no more
4337                                  * away then 1/2 our running total
4338                                  * so factor it in.
4339                                  */
4340                                 addpart = bytes_ps * utim;
4341                                 addpart /= (srtt * 8);
4342                         } else {
4343                                 /*
4344                                  * Don't allow a single measurement
4345                                  * to account for more than 1/2 of the
4346                                  * WMA. This could happen on a retransmission
4347                                  * where utim becomes huge compared to
4348                                  * srtt (multiple retransmissions when using
4349                                  * the sending rate which factors in all the
4350                                  * transmissions from the first one).
4351                                  */
4352                                 subpart = rack->r_ctl.gp_bw / 2;
4353                                 addpart = bytes_ps / 2;
4354                         }
4355                         resid_bw = rack->r_ctl.gp_bw - subpart;
4356                         rack->r_ctl.gp_bw = resid_bw + addpart;
4357                         did_add = 1;
4358                 } else {
4359                         if ((utim / srtt) <= 1) {
4360                                 /*
4361                                  * The b/w update was over a small period
4362                                  * of time. The idea here is to prevent a small
4363                                  * measurement time period from counting
4364                                  * too much. So we scale it based on the
4365                                  * time so it attributes less than 1/rack_wma_divisor
4366                                  * of its measurement.
4367                                  */
4368                                 subpart = rack->r_ctl.gp_bw * utim;
4369                                 subpart /= (srtt * rack_wma_divisor);
4370                                 addpart = bytes_ps * utim;
4371                                 addpart /= (srtt * rack_wma_divisor);
4372                         } else {
4373                                 /*
4374                                  * The scaled measurement was long
4375                                  * enough so lets just add in the
4376                                  * portion of the measurement i.e. 1/rack_wma_divisor
4377                                  */
4378                                 subpart = rack->r_ctl.gp_bw / rack_wma_divisor;
4379                                 addpart = bytes_ps / rack_wma_divisor;
4380                         }
4381                         if ((rack->measure_saw_probe_rtt == 0) ||
4382                             (bytes_ps > rack->r_ctl.gp_bw)) {
4383                                 /*
4384                                  * For probe-rtt we only add it in
4385                                  * if its larger, all others we just
4386                                  * add in.
4387                                  */
4388                                 did_add = 1;
4389                                 resid_bw = rack->r_ctl.gp_bw - subpart;
4390                                 rack->r_ctl.gp_bw = resid_bw + addpart;
4391                         }
4392                 }
4393         }
4394         if ((rack->gp_ready == 0) &&
4395             (rack->r_ctl.num_measurements >= rack->r_ctl.req_measurements)) {
4396                 /* We have enough measurements now */
4397                 rack->gp_ready = 1;
4398                 rack_set_cc_pacing(rack);
4399                 if (rack->defer_options)
4400                         rack_apply_deferred_options(rack);
4401         }
4402         rack_log_pacing_delay_calc(rack, subpart, addpart, bytes_ps, stim,
4403                                    rack_get_bw(rack), 22, did_add, NULL, quality);
4404         /* We do not update any multipliers if we are in or have seen a probe-rtt */
4405         if ((rack->measure_saw_probe_rtt == 0) && rack->rc_gp_rtt_set)
4406                 rack_update_multiplier(rack, timely_says, bytes_ps,
4407                                        rack->r_ctl.rc_gp_srtt,
4408                                        rack->r_ctl.rc_rtt_diff);
4409         rack_log_pacing_delay_calc(rack, bytes, tim, bytes_ps, stim,
4410                                    rack_get_bw(rack), 3, line, NULL, quality);
4411         /* reset the gp srtt and setup the new prev */
4412         rack->r_ctl.rc_prev_gp_srtt = rack->r_ctl.rc_gp_srtt;
4413         /* Record the lost count for the next measurement */
4414         rack->r_ctl.rc_loss_at_start = rack->r_ctl.rc_loss_count;
4415         /*
4416          * We restart our diffs based on the gpsrtt in the
4417          * measurement window.
4418          */
4419         rack->rc_gp_rtt_set = 0;
4420         rack->rc_gp_saw_rec = 0;
4421         rack->rc_gp_saw_ca = 0;
4422         rack->rc_gp_saw_ss = 0;
4423         rack->rc_dragged_bottom = 0;
4424 skip_measurement:
4425
4426 #ifdef STATS
4427         stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT,
4428                                  gput);
4429         /*
4430          * XXXLAS: This is a temporary hack, and should be
4431          * chained off VOI_TCP_GPUT when stats(9) grows an
4432          * API to deal with chained VOIs.
4433          */
4434         if (tp->t_stats_gput_prev > 0)
4435                 stats_voi_update_abs_s32(tp->t_stats,
4436                                          VOI_TCP_GPUT_ND,
4437                                          ((gput - tp->t_stats_gput_prev) * 100) /
4438                                          tp->t_stats_gput_prev);
4439 #endif
4440         tp->t_flags &= ~TF_GPUTINPROG;
4441         tp->t_stats_gput_prev = gput;
4442         /*
4443          * Now are we app limited now and there is space from where we
4444          * were to where we want to go?
4445          *
4446          * We don't do the other case i.e. non-applimited here since
4447          * the next send will trigger us picking up the missing data.
4448          */
4449         if (rack->r_ctl.rc_first_appl &&
4450             TCPS_HAVEESTABLISHED(tp->t_state) &&
4451             rack->r_ctl.rc_app_limited_cnt &&
4452             (SEQ_GT(rack->r_ctl.rc_first_appl->r_start, th_ack)) &&
4453             ((rack->r_ctl.rc_first_appl->r_end - th_ack) >
4454              max(rc_init_window(rack), (MIN_GP_WIN * segsiz)))) {
4455                 /*
4456                  * Yep there is enough outstanding to make a measurement here.
4457                  */
4458                 struct rack_sendmap *rsm, fe;
4459
4460                 rack->r_ctl.rc_gp_lowrtt = 0xffffffff;
4461                 rack->r_ctl.rc_gp_high_rwnd = rack->rc_tp->snd_wnd;
4462                 tp->gput_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time);
4463                 rack->app_limited_needs_set = 0;
4464                 tp->gput_seq = th_ack;
4465                 if (rack->in_probe_rtt)
4466                         rack->measure_saw_probe_rtt = 1;
4467                 else if ((rack->measure_saw_probe_rtt) &&
4468                          (SEQ_GEQ(tp->gput_seq, rack->r_ctl.rc_probertt_sndmax_atexit)))
4469                         rack->measure_saw_probe_rtt = 0;
4470                 if ((rack->r_ctl.rc_first_appl->r_end - th_ack) >= rack_get_measure_window(tp, rack)) {
4471                         /* There is a full window to gain info from */
4472                         tp->gput_ack = tp->gput_seq + rack_get_measure_window(tp, rack);
4473                 } else {
4474                         /* We can only measure up to the applimited point */
4475                         tp->gput_ack = tp->gput_seq + (rack->r_ctl.rc_first_appl->r_end - th_ack);
4476                         if ((tp->gput_ack - tp->gput_seq) < (MIN_GP_WIN * segsiz)) {
4477                                 /*
4478                                  * We don't have enough to make a measurement.
4479                                  */
4480                                 tp->t_flags &= ~TF_GPUTINPROG;
4481                                 rack_log_pacing_delay_calc(rack, tp->gput_ack, tp->gput_seq,
4482                                                            0, 0, 0, 6, __LINE__, NULL, quality);
4483                                 return;
4484                         }
4485                 }
4486                 if (tp->t_state >= TCPS_FIN_WAIT_1) {
4487                         /*
4488                          * We will get no more data into the SB
4489                          * this means we need to have the data available
4490                          * before we start a measurement.
4491                          */
4492                         if (sbavail(&tp->t_inpcb->inp_socket->so_snd) < (tp->gput_ack - tp->gput_seq)) {
4493                                 /* Nope not enough data. */
4494                                 return;
4495                         }
4496                 }
4497                 tp->t_flags |= TF_GPUTINPROG;
4498                 /*
4499                  * Now we need to find the timestamp of the send at tp->gput_seq
4500                  * for the send based measurement.
4501                  */
4502                 fe.r_start = tp->gput_seq;
4503                 rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe);
4504                 if (rsm) {
4505                         /* Ok send-based limit is set */
4506                         if (SEQ_LT(rsm->r_start, tp->gput_seq)) {
4507                                 /*
4508                                  * Move back to include the earlier part
4509                                  * so our ack time lines up right (this may
4510                                  * make an overlapping measurement but thats
4511                                  * ok).
4512                                  */
4513                                 tp->gput_seq = rsm->r_start;
4514                         }
4515                         if (rsm->r_flags & RACK_ACKED)
4516                                 tp->gput_ts = (uint32_t)rsm->r_ack_arrival;
4517                         else
4518                                 rack->app_limited_needs_set = 1;
4519                         rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
4520                 } else {
4521                         /*
4522                          * If we don't find the rsm due to some
4523                          * send-limit set the current time, which
4524                          * basically disables the send-limit.
4525                          */
4526                         struct timeval tv;
4527
4528                         microuptime(&tv);
4529                         rack->r_ctl.rc_gp_output_ts = rack_to_usec_ts(&tv);
4530                 }
4531                 rack_log_pacing_delay_calc(rack,
4532                                            tp->gput_seq,
4533                                            tp->gput_ack,
4534                                            (uint64_t)rsm,
4535                                            tp->gput_ts,
4536                                            rack->r_ctl.rc_app_limited_cnt,
4537                                            9,
4538                                            __LINE__, NULL, quality);
4539         }
4540 }
4541
4542 /*
4543  * CC wrapper hook functions
4544  */
4545 static void
4546 rack_ack_received(struct tcpcb *tp, struct tcp_rack *rack, uint32_t th_ack, uint16_t nsegs,
4547     uint16_t type, int32_t recovery)
4548 {
4549         uint32_t prior_cwnd, acked;
4550         struct tcp_log_buffer *lgb = NULL;
4551         uint8_t labc_to_use, quality;
4552
4553         INP_WLOCK_ASSERT(tp->t_inpcb);
4554         tp->ccv->nsegs = nsegs;
4555         acked = tp->ccv->bytes_this_ack = (th_ack - tp->snd_una);
4556         if ((recovery) && (rack->r_ctl.rc_early_recovery_segs)) {
4557                 uint32_t max;
4558
4559                 max = rack->r_ctl.rc_early_recovery_segs * ctf_fixed_maxseg(tp);
4560                 if (tp->ccv->bytes_this_ack > max) {
4561                         tp->ccv->bytes_this_ack = max;
4562                 }
4563         }
4564 #ifdef STATS
4565         stats_voi_update_abs_s32(tp->t_stats, VOI_TCP_CALCFRWINDIFF,
4566             ((int32_t)rack->r_ctl.cwnd_to_use) - tp->snd_wnd);
4567 #endif
4568         quality = RACK_QUALITY_NONE;
4569         if ((tp->t_flags & TF_GPUTINPROG) &&
4570             rack_enough_for_measurement(tp, rack, th_ack, &quality)) {
4571                 /* Measure the Goodput */
4572                 rack_do_goodput_measurement(tp, rack, th_ack, __LINE__, quality);
4573 #ifdef NETFLIX_PEAKRATE
4574                 if ((type == CC_ACK) &&
4575                     (tp->t_maxpeakrate)) {
4576                         /*
4577                          * We update t_peakrate_thr. This gives us roughly
4578                          * one update per round trip time. Note
4579                          * it will only be used if pace_always is off i.e
4580                          * we don't do this for paced flows.
4581                          */
4582                         rack_update_peakrate_thr(tp);
4583                 }
4584 #endif
4585         }
4586         /* Which way our we limited, if not cwnd limited no advance in CA */
4587         if (tp->snd_cwnd <= tp->snd_wnd)
4588                 tp->ccv->flags |= CCF_CWND_LIMITED;
4589         else
4590                 tp->ccv->flags &= ~CCF_CWND_LIMITED;
4591         if (tp->snd_cwnd > tp->snd_ssthresh) {
4592                 tp->t_bytes_acked += min(tp->ccv->bytes_this_ack,
4593                          nsegs * V_tcp_abc_l_var * ctf_fixed_maxseg(tp));
4594                 /* For the setting of a window past use the actual scwnd we are using */
4595                 if (tp->t_bytes_acked >= rack->r_ctl.cwnd_to_use) {
4596                         tp->t_bytes_acked -= rack->r_ctl.cwnd_to_use;
4597                         tp->ccv->flags |= CCF_ABC_SENTAWND;
4598                 }
4599         } else {
4600                 tp->ccv->flags &= ~CCF_ABC_SENTAWND;
4601                 tp->t_bytes_acked = 0;
4602         }
4603         prior_cwnd = tp->snd_cwnd;
4604         if ((recovery == 0) || (rack_max_abc_post_recovery == 0) || rack->r_use_labc_for_rec ||
4605             (rack_client_low_buf && (rack->client_bufferlvl < rack_client_low_buf)))
4606                 labc_to_use = rack->rc_labc;
4607         else
4608                 labc_to_use = rack_max_abc_post_recovery;
4609         if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
4610                 union tcp_log_stackspecific log;
4611                 struct timeval tv;
4612
4613                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
4614                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
4615                 log.u_bbr.flex1 = th_ack;
4616                 log.u_bbr.flex2 = tp->ccv->flags;
4617                 log.u_bbr.flex3 = tp->ccv->bytes_this_ack;
4618                 log.u_bbr.flex4 = tp->ccv->nsegs;
4619                 log.u_bbr.flex5 = labc_to_use;
4620                 log.u_bbr.flex6 = prior_cwnd;
4621                 log.u_bbr.flex7 = V_tcp_do_newsack;
4622                 log.u_bbr.flex8 = 1;
4623                 lgb = tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
4624                                      0, &log, false, NULL, NULL, 0, &tv);
4625         }
4626         if (CC_ALGO(tp)->ack_received != NULL) {
4627                 /* XXXLAS: Find a way to live without this */
4628                 tp->ccv->curack = th_ack;
4629                 tp->ccv->labc = labc_to_use;
4630                 tp->ccv->flags |= CCF_USE_LOCAL_ABC;
4631                 CC_ALGO(tp)->ack_received(tp->ccv, type);
4632         }
4633         if (lgb) {
4634                 lgb->tlb_stackinfo.u_bbr.flex6 = tp->snd_cwnd;
4635         }
4636         if (rack->r_must_retran) {
4637                 if (SEQ_GEQ(th_ack, rack->r_ctl.rc_snd_max_at_rto)) {
4638                         /*
4639                          * We now are beyond the rxt point so lets disable
4640                          * the flag.
4641                          */
4642                         rack->r_ctl.rc_out_at_rto = 0;
4643                         rack->r_must_retran = 0;
4644                 } else if ((prior_cwnd + ctf_fixed_maxseg(tp)) <= tp->snd_cwnd) {
4645                         /*
4646                          * Only decrement the rc_out_at_rto if the cwnd advances
4647                          * at least a whole segment. Otherwise next time the peer
4648                          * acks, we won't be able to send this generaly happens
4649                          * when we are in Congestion Avoidance.
4650                          */
4651                         if (acked <= rack->r_ctl.rc_out_at_rto){
4652                                 rack->r_ctl.rc_out_at_rto -= acked;
4653                         } else {
4654                                 rack->r_ctl.rc_out_at_rto = 0;
4655                         }
4656                 }
4657         }
4658 #ifdef STATS
4659         stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_LCWIN, rack->r_ctl.cwnd_to_use);
4660 #endif
4661         if (rack->r_ctl.rc_rack_largest_cwnd < rack->r_ctl.cwnd_to_use) {
4662                 rack->r_ctl.rc_rack_largest_cwnd = rack->r_ctl.cwnd_to_use;
4663         }
4664 #ifdef NETFLIX_PEAKRATE
4665         /* we enforce max peak rate if it is set and we are not pacing */
4666         if ((rack->rc_always_pace == 0) &&
4667             tp->t_peakrate_thr &&
4668             (tp->snd_cwnd > tp->t_peakrate_thr)) {
4669                 tp->snd_cwnd = tp->t_peakrate_thr;
4670         }
4671 #endif
4672 }
4673
4674 static void
4675 tcp_rack_partialack(struct tcpcb *tp)
4676 {
4677         struct tcp_rack *rack;
4678
4679         rack = (struct tcp_rack *)tp->t_fb_ptr;
4680         INP_WLOCK_ASSERT(tp->t_inpcb);
4681         /*
4682          * If we are doing PRR and have enough
4683          * room to send <or> we are pacing and prr
4684          * is disabled we will want to see if we
4685          * can send data (by setting r_wanted_output to
4686          * true).
4687          */
4688         if ((rack->r_ctl.rc_prr_sndcnt > 0) ||
4689             rack->rack_no_prr)
4690                 rack->r_wanted_output = 1;
4691 }
4692
4693 static void
4694 rack_post_recovery(struct tcpcb *tp, uint32_t th_ack)
4695 {
4696         struct tcp_rack *rack;
4697         uint32_t orig_cwnd;
4698
4699         orig_cwnd = tp->snd_cwnd;
4700         INP_WLOCK_ASSERT(tp->t_inpcb);
4701         rack = (struct tcp_rack *)tp->t_fb_ptr;
4702         /* only alert CC if we alerted when we entered */
4703         if (CC_ALGO(tp)->post_recovery != NULL) {
4704                 tp->ccv->curack = th_ack;
4705                 CC_ALGO(tp)->post_recovery(tp->ccv);
4706                 if (tp->snd_cwnd < tp->snd_ssthresh) {
4707                         /*
4708                          * Rack has burst control and pacing
4709                          * so lets not set this any lower than
4710                          * snd_ssthresh per RFC-6582 (option 2).
4711                          */
4712                         tp->snd_cwnd = tp->snd_ssthresh;
4713                 }
4714         }
4715         if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
4716                 union tcp_log_stackspecific log;
4717                 struct timeval tv;
4718
4719                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
4720                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
4721                 log.u_bbr.flex1 = th_ack;
4722                 log.u_bbr.flex2 = tp->ccv->flags;
4723                 log.u_bbr.flex3 = tp->ccv->bytes_this_ack;
4724                 log.u_bbr.flex4 = tp->ccv->nsegs;
4725                 log.u_bbr.flex5 = V_tcp_abc_l_var;
4726                 log.u_bbr.flex6 = orig_cwnd;
4727                 log.u_bbr.flex7 = V_tcp_do_newsack;
4728                 log.u_bbr.pkts_out = rack->r_ctl.rc_prr_sndcnt;
4729                 log.u_bbr.flex8 = 2;
4730                 tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
4731                                0, &log, false, NULL, NULL, 0, &tv);
4732         }
4733         if ((rack->rack_no_prr == 0) &&
4734             (rack->no_prr_addback == 0) &&
4735             (rack->r_ctl.rc_prr_sndcnt > 0)) {
4736                 /*
4737                  * Suck the next prr cnt back into cwnd, but
4738                  * only do that if we are not application limited.
4739                  */
4740                 if (ctf_outstanding(tp) <= sbavail(&(tp->t_inpcb->inp_socket->so_snd))) {
4741                         /*
4742                          * We are allowed to add back to the cwnd the amount we did
4743                          * not get out if:
4744                          * a) no_prr_addback is off.
4745                          * b) we are not app limited
4746                          * c) we are doing prr
4747                          * <and>
4748                          * d) it is bounded by rack_prr_addbackmax (if addback is 0, then none).
4749                          */
4750                         tp->snd_cwnd += min((ctf_fixed_maxseg(tp) * rack_prr_addbackmax),
4751                                             rack->r_ctl.rc_prr_sndcnt);
4752                 }
4753                 rack->r_ctl.rc_prr_sndcnt = 0;
4754                 rack_log_to_prr(rack, 1, 0, __LINE__);
4755         }
4756         rack_log_to_prr(rack, 14, orig_cwnd, __LINE__);
4757         tp->snd_recover = tp->snd_una;
4758         if (rack->r_ctl.dsack_persist) {
4759                 rack->r_ctl.dsack_persist--;
4760                 if (rack->r_ctl.num_dsack && (rack->r_ctl.dsack_persist == 0)) {
4761                         rack->r_ctl.num_dsack = 0;
4762                 }
4763                 rack_log_dsack_event(rack, 1, __LINE__, 0, 0);
4764         }
4765         EXIT_RECOVERY(tp->t_flags);
4766 }
4767
4768 static void
4769 rack_cong_signal(struct tcpcb *tp, uint32_t type, uint32_t ack, int line)
4770 {
4771         struct tcp_rack *rack;
4772         uint32_t ssthresh_enter, cwnd_enter, in_rec_at_entry, orig_cwnd;
4773
4774         INP_WLOCK_ASSERT(tp->t_inpcb);
4775 #ifdef STATS
4776         stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_CSIG, type);
4777 #endif
4778         if (IN_RECOVERY(tp->t_flags) == 0) {
4779                 in_rec_at_entry = 0;
4780                 ssthresh_enter = tp->snd_ssthresh;
4781                 cwnd_enter = tp->snd_cwnd;
4782         } else
4783                 in_rec_at_entry = 1;
4784         rack = (struct tcp_rack *)tp->t_fb_ptr;
4785         switch (type) {
4786         case CC_NDUPACK:
4787                 tp->t_flags &= ~TF_WASFRECOVERY;
4788                 tp->t_flags &= ~TF_WASCRECOVERY;
4789                 if (!IN_FASTRECOVERY(tp->t_flags)) {
4790                         rack->r_ctl.rc_prr_delivered = 0;
4791                         rack->r_ctl.rc_prr_out = 0;
4792                         if (rack->rack_no_prr == 0) {
4793                                 rack->r_ctl.rc_prr_sndcnt = ctf_fixed_maxseg(tp);
4794                                 rack_log_to_prr(rack, 2, in_rec_at_entry, line);
4795                         }
4796                         rack->r_ctl.rc_prr_recovery_fs = tp->snd_max - tp->snd_una;
4797                         tp->snd_recover = tp->snd_max;
4798                         if (tp->t_flags2 & TF2_ECN_PERMIT)
4799                                 tp->t_flags2 |= TF2_ECN_SND_CWR;
4800                 }
4801                 break;
4802         case CC_ECN:
4803                 if (!IN_CONGRECOVERY(tp->t_flags) ||
4804                     /*
4805                      * Allow ECN reaction on ACK to CWR, if
4806                      * that data segment was also CE marked.
4807                      */
4808                     SEQ_GEQ(ack, tp->snd_recover)) {
4809                         EXIT_CONGRECOVERY(tp->t_flags);
4810                         KMOD_TCPSTAT_INC(tcps_ecn_rcwnd);
4811                         tp->snd_recover = tp->snd_max + 1;
4812                         if (tp->t_flags2 & TF2_ECN_PERMIT)
4813                                 tp->t_flags2 |= TF2_ECN_SND_CWR;
4814                 }
4815                 break;
4816         case CC_RTO:
4817                 tp->t_dupacks = 0;
4818                 tp->t_bytes_acked = 0;
4819                 EXIT_RECOVERY(tp->t_flags);
4820                 tp->snd_ssthresh = max(2, min(tp->snd_wnd, rack->r_ctl.cwnd_to_use) / 2 /
4821                     ctf_fixed_maxseg(tp)) * ctf_fixed_maxseg(tp);
4822                 orig_cwnd = tp->snd_cwnd;
4823                 tp->snd_cwnd = ctf_fixed_maxseg(tp);
4824                 rack_log_to_prr(rack, 16, orig_cwnd, line);
4825                 if (tp->t_flags2 & TF2_ECN_PERMIT)
4826                         tp->t_flags2 |= TF2_ECN_SND_CWR;
4827                 break;
4828         case CC_RTO_ERR:
4829                 KMOD_TCPSTAT_INC(tcps_sndrexmitbad);
4830                 /* RTO was unnecessary, so reset everything. */
4831                 tp->snd_cwnd = tp->snd_cwnd_prev;
4832                 tp->snd_ssthresh = tp->snd_ssthresh_prev;
4833                 tp->snd_recover = tp->snd_recover_prev;
4834                 if (tp->t_flags & TF_WASFRECOVERY) {
4835                         ENTER_FASTRECOVERY(tp->t_flags);
4836                         tp->t_flags &= ~TF_WASFRECOVERY;
4837                 }
4838                 if (tp->t_flags & TF_WASCRECOVERY) {
4839                         ENTER_CONGRECOVERY(tp->t_flags);
4840                         tp->t_flags &= ~TF_WASCRECOVERY;
4841                 }
4842                 tp->snd_nxt = tp->snd_max;
4843                 tp->t_badrxtwin = 0;
4844                 break;
4845         }
4846         if ((CC_ALGO(tp)->cong_signal != NULL)  &&
4847             (type != CC_RTO)){
4848                 tp->ccv->curack = ack;
4849                 CC_ALGO(tp)->cong_signal(tp->ccv, type);
4850         }
4851         if ((in_rec_at_entry == 0) && IN_RECOVERY(tp->t_flags)) {
4852                 rack_log_to_prr(rack, 15, cwnd_enter, line);
4853                 rack->r_ctl.dsack_byte_cnt = 0;
4854                 rack->r_ctl.retran_during_recovery = 0;
4855                 rack->r_ctl.rc_cwnd_at_erec = cwnd_enter;
4856                 rack->r_ctl.rc_ssthresh_at_erec = ssthresh_enter;
4857                 rack->r_ent_rec_ns = 1;
4858         }
4859 }
4860
4861 static inline void
4862 rack_cc_after_idle(struct tcp_rack *rack, struct tcpcb *tp)
4863 {
4864         uint32_t i_cwnd;
4865
4866         INP_WLOCK_ASSERT(tp->t_inpcb);
4867
4868 #ifdef NETFLIX_STATS
4869         KMOD_TCPSTAT_INC(tcps_idle_restarts);
4870         if (tp->t_state == TCPS_ESTABLISHED)
4871                 KMOD_TCPSTAT_INC(tcps_idle_estrestarts);
4872 #endif
4873         if (CC_ALGO(tp)->after_idle != NULL)
4874                 CC_ALGO(tp)->after_idle(tp->ccv);
4875
4876         if (tp->snd_cwnd == 1)
4877                 i_cwnd = tp->t_maxseg;          /* SYN(-ACK) lost */
4878         else
4879                 i_cwnd = rc_init_window(rack);
4880
4881         /*
4882          * Being idle is no different than the initial window. If the cc
4883          * clamps it down below the initial window raise it to the initial
4884          * window.
4885          */
4886         if (tp->snd_cwnd < i_cwnd) {
4887                 tp->snd_cwnd = i_cwnd;
4888         }
4889 }
4890
4891 /*
4892  * Indicate whether this ack should be delayed.  We can delay the ack if
4893  * following conditions are met:
4894  *      - There is no delayed ack timer in progress.
4895  *      - Our last ack wasn't a 0-sized window. We never want to delay
4896  *        the ack that opens up a 0-sized window.
4897  *      - LRO wasn't used for this segment. We make sure by checking that the
4898  *        segment size is not larger than the MSS.
4899  *      - Delayed acks are enabled or this is a half-synchronized T/TCP
4900  *        connection.
4901  */
4902 #define DELAY_ACK(tp, tlen)                      \
4903         (((tp->t_flags & TF_RXWIN0SENT) == 0) && \
4904         ((tp->t_flags & TF_DELACK) == 0) &&      \
4905         (tlen <= tp->t_maxseg) &&                \
4906         (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
4907
4908 static struct rack_sendmap *
4909 rack_find_lowest_rsm(struct tcp_rack *rack)
4910 {
4911         struct rack_sendmap *rsm;
4912
4913         /*
4914          * Walk the time-order transmitted list looking for an rsm that is
4915          * not acked. This will be the one that was sent the longest time
4916          * ago that is still outstanding.
4917          */
4918         TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) {
4919                 if (rsm->r_flags & RACK_ACKED) {
4920                         continue;
4921                 }
4922                 goto finish;
4923         }
4924 finish:
4925         return (rsm);
4926 }
4927
4928 static struct rack_sendmap *
4929 rack_find_high_nonack(struct tcp_rack *rack, struct rack_sendmap *rsm)
4930 {
4931         struct rack_sendmap *prsm;
4932
4933         /*
4934          * Walk the sequence order list backward until we hit and arrive at
4935          * the highest seq not acked. In theory when this is called it
4936          * should be the last segment (which it was not).
4937          */
4938         prsm = rsm;
4939         RB_FOREACH_REVERSE_FROM(prsm, rack_rb_tree_head, rsm) {
4940                 if (prsm->r_flags & (RACK_ACKED | RACK_HAS_FIN)) {
4941                         continue;
4942                 }
4943                 return (prsm);
4944         }
4945         return (NULL);
4946 }
4947
4948 static uint32_t
4949 rack_calc_thresh_rack(struct tcp_rack *rack, uint32_t srtt, uint32_t cts)
4950 {
4951         int32_t lro;
4952         uint32_t thresh;
4953
4954         /*
4955          * lro is the flag we use to determine if we have seen reordering.
4956          * If it gets set we have seen reordering. The reorder logic either
4957          * works in one of two ways:
4958          *
4959          * If reorder-fade is configured, then we track the last time we saw
4960          * re-ordering occur. If we reach the point where enough time as
4961          * passed we no longer consider reordering has occuring.
4962          *
4963          * Or if reorder-face is 0, then once we see reordering we consider
4964          * the connection to alway be subject to reordering and just set lro
4965          * to 1.
4966          *
4967          * In the end if lro is non-zero we add the extra time for
4968          * reordering in.
4969          */
4970         if (srtt == 0)
4971                 srtt = 1;
4972         if (rack->r_ctl.rc_reorder_ts) {
4973                 if (rack->r_ctl.rc_reorder_fade) {
4974                         if (SEQ_GEQ(cts, rack->r_ctl.rc_reorder_ts)) {
4975                                 lro = cts - rack->r_ctl.rc_reorder_ts;
4976                                 if (lro == 0) {
4977                                         /*
4978                                          * No time as passed since the last
4979                                          * reorder, mark it as reordering.
4980                                          */
4981                                         lro = 1;
4982                                 }
4983                         } else {
4984                                 /* Negative time? */
4985                                 lro = 0;
4986                         }
4987                         if (lro > rack->r_ctl.rc_reorder_fade) {
4988                                 /* Turn off reordering seen too */
4989                                 rack->r_ctl.rc_reorder_ts = 0;
4990                                 lro = 0;
4991                         }
4992                 } else {
4993                         /* Reodering does not fade */
4994                         lro = 1;
4995                 }
4996         } else {
4997                 lro = 0;
4998         }
4999         if (rack->rc_rack_tmr_std_based == 0) {
5000                 thresh = srtt + rack->r_ctl.rc_pkt_delay;
5001         } else {
5002                 /* Standards based pkt-delay is 1/4 srtt */
5003                 thresh = srtt +  (srtt >> 2);
5004         }
5005         if (lro && (rack->rc_rack_tmr_std_based == 0)) {
5006                 /* It must be set, if not you get 1/4 rtt */
5007                 if (rack->r_ctl.rc_reorder_shift)
5008                         thresh += (srtt >> rack->r_ctl.rc_reorder_shift);
5009                 else
5010                         thresh += (srtt >> 2);
5011         }
5012         if (rack->rc_rack_use_dsack &&
5013             lro &&
5014             (rack->r_ctl.num_dsack > 0)) {
5015                 /*
5016                  * We only increase the reordering window if we
5017                  * have seen reordering <and> we have a DSACK count.
5018                  */
5019                 thresh += rack->r_ctl.num_dsack * (srtt >> 2);
5020                 rack_log_dsack_event(rack, 4, __LINE__, srtt, thresh);
5021         }
5022         /* SRTT * 2 is the ceiling */
5023         if (thresh > (srtt * 2)) {
5024                 thresh = srtt * 2;
5025         }
5026         /* And we don't want it above the RTO max either */
5027         if (thresh > rack_rto_max) {
5028                 thresh = rack_rto_max;
5029         }
5030         rack_log_dsack_event(rack, 6, __LINE__, srtt, thresh);
5031         return (thresh);
5032 }
5033
5034 static uint32_t
5035 rack_calc_thresh_tlp(struct tcpcb *tp, struct tcp_rack *rack,
5036                      struct rack_sendmap *rsm, uint32_t srtt)
5037 {
5038         struct rack_sendmap *prsm;
5039         uint32_t thresh, len;
5040         int segsiz;
5041
5042         if (srtt == 0)
5043                 srtt = 1;
5044         if (rack->r_ctl.rc_tlp_threshold)
5045                 thresh = srtt + (srtt / rack->r_ctl.rc_tlp_threshold);
5046         else
5047                 thresh = (srtt * 2);
5048
5049         /* Get the previous sent packet, if any */
5050         segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
5051         len = rsm->r_end - rsm->r_start;
5052         if (rack->rack_tlp_threshold_use == TLP_USE_ID) {
5053                 /* Exactly like the ID */
5054                 if (((tp->snd_max - tp->snd_una) - rack->r_ctl.rc_sacked + rack->r_ctl.rc_holes_rxt) <= segsiz) {
5055                         uint32_t alt_thresh;
5056                         /*
5057                          * Compensate for delayed-ack with the d-ack time.
5058                          */
5059                         alt_thresh = srtt + (srtt / 2) + rack_delayed_ack_time;
5060                         if (alt_thresh > thresh)
5061                                 thresh = alt_thresh;
5062                 }
5063         } else if (rack->rack_tlp_threshold_use == TLP_USE_TWO_ONE) {
5064                 /* 2.1 behavior */
5065                 prsm = TAILQ_PREV(rsm, rack_head, r_tnext);
5066                 if (prsm && (len <= segsiz)) {
5067                         /*
5068                          * Two packets outstanding, thresh should be (2*srtt) +
5069                          * possible inter-packet delay (if any).
5070                          */
5071                         uint32_t inter_gap = 0;
5072                         int idx, nidx;
5073
5074                         idx = rsm->r_rtr_cnt - 1;
5075                         nidx = prsm->r_rtr_cnt - 1;
5076                         if (rsm->r_tim_lastsent[nidx] >= prsm->r_tim_lastsent[idx]) {
5077                                 /* Yes it was sent later (or at the same time) */
5078                                 inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx];
5079                         }
5080                         thresh += inter_gap;
5081                 } else if (len <= segsiz) {
5082                         /*
5083                          * Possibly compensate for delayed-ack.
5084                          */
5085                         uint32_t alt_thresh;
5086
5087                         alt_thresh = srtt + (srtt / 2) + rack_delayed_ack_time;
5088                         if (alt_thresh > thresh)
5089                                 thresh = alt_thresh;
5090                 }
5091         } else if (rack->rack_tlp_threshold_use == TLP_USE_TWO_TWO) {
5092                 /* 2.2 behavior */
5093                 if (len <= segsiz) {
5094                         uint32_t alt_thresh;
5095                         /*
5096                          * Compensate for delayed-ack with the d-ack time.
5097                          */
5098                         alt_thresh = srtt + (srtt / 2) + rack_delayed_ack_time;
5099                         if (alt_thresh > thresh)
5100                                 thresh = alt_thresh;
5101                 }
5102         }
5103         /* Not above an RTO */
5104         if (thresh > tp->t_rxtcur) {
5105                 thresh = tp->t_rxtcur;
5106         }
5107         /* Not above a RTO max */
5108         if (thresh > rack_rto_max) {
5109                 thresh = rack_rto_max;
5110         }
5111         /* Apply user supplied min TLP */
5112         if (thresh < rack_tlp_min) {
5113                 thresh = rack_tlp_min;
5114         }
5115         return (thresh);
5116 }
5117
5118 static uint32_t
5119 rack_grab_rtt(struct tcpcb *tp, struct tcp_rack *rack)
5120 {
5121         /*
5122          * We want the rack_rtt which is the
5123          * last rtt we measured. However if that
5124          * does not exist we fallback to the srtt (which
5125          * we probably will never do) and then as a last
5126          * resort we use RACK_INITIAL_RTO if no srtt is
5127          * yet set.
5128          */
5129         if (rack->rc_rack_rtt)
5130                 return (rack->rc_rack_rtt);
5131         else if (tp->t_srtt == 0)
5132                 return (RACK_INITIAL_RTO);
5133         return (tp->t_srtt);
5134 }
5135
5136 static struct rack_sendmap *
5137 rack_check_recovery_mode(struct tcpcb *tp, uint32_t tsused)
5138 {
5139         /*
5140          * Check to see that we don't need to fall into recovery. We will
5141          * need to do so if our oldest transmit is past the time we should
5142          * have had an ack.
5143          */
5144         struct tcp_rack *rack;
5145         struct rack_sendmap *rsm;
5146         int32_t idx;
5147         uint32_t srtt, thresh;
5148
5149         rack = (struct tcp_rack *)tp->t_fb_ptr;
5150         if (RB_EMPTY(&rack->r_ctl.rc_mtree)) {
5151                 return (NULL);
5152         }
5153         rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
5154         if (rsm == NULL)
5155                 return (NULL);
5156
5157
5158         if (rsm->r_flags & RACK_ACKED) {
5159                 rsm = rack_find_lowest_rsm(rack);
5160                 if (rsm == NULL)
5161                         return (NULL);
5162         }
5163         idx = rsm->r_rtr_cnt - 1;
5164         srtt = rack_grab_rtt(tp, rack);
5165         thresh = rack_calc_thresh_rack(rack, srtt, tsused);
5166         if (TSTMP_LT(tsused, ((uint32_t)rsm->r_tim_lastsent[idx]))) {
5167                 return (NULL);
5168         }
5169         if ((tsused - ((uint32_t)rsm->r_tim_lastsent[idx])) < thresh) {
5170                 return (NULL);
5171         }
5172         /* Ok if we reach here we are over-due and this guy can be sent */
5173         rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__);
5174         return (rsm);
5175 }
5176
5177 static uint32_t
5178 rack_get_persists_timer_val(struct tcpcb *tp, struct tcp_rack *rack)
5179 {
5180         int32_t t;
5181         int32_t tt;
5182         uint32_t ret_val;
5183
5184         t = (tp->t_srtt + (tp->t_rttvar << 2));
5185         RACK_TCPT_RANGESET(tt, t * tcp_backoff[tp->t_rxtshift],
5186             rack_persist_min, rack_persist_max, rack->r_ctl.timer_slop);
5187         rack->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT;
5188         ret_val = (uint32_t)tt;
5189         return (ret_val);
5190 }
5191
5192 static uint32_t
5193 rack_timer_start(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int sup_rack)
5194 {
5195         /*
5196          * Start the FR timer, we do this based on getting the first one in
5197          * the rc_tmap. Note that if its NULL we must stop the timer. in all
5198          * events we need to stop the running timer (if its running) before
5199          * starting the new one.
5200          */
5201         uint32_t thresh, exp, to, srtt, time_since_sent, tstmp_touse;
5202         uint32_t srtt_cur;
5203         int32_t idx;
5204         int32_t is_tlp_timer = 0;
5205         struct rack_sendmap *rsm;
5206
5207         if (rack->t_timers_stopped) {
5208                 /* All timers have been stopped none are to run */
5209                 return (0);
5210         }
5211         if (rack->rc_in_persist) {
5212                 /* We can't start any timer in persists */
5213                 return (rack_get_persists_timer_val(tp, rack));
5214         }
5215         rack->rc_on_min_to = 0;
5216         if ((tp->t_state < TCPS_ESTABLISHED) ||
5217             ((tp->t_flags & TF_SACK_PERMIT) == 0)) {
5218                 goto activate_rxt;
5219         }
5220         rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
5221         if ((rsm == NULL) || sup_rack) {
5222                 /* Nothing on the send map or no rack */
5223 activate_rxt:
5224                 time_since_sent = 0;
5225                 rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
5226                 if (rsm) {
5227                         /*
5228                          * Should we discount the RTX timer any?
5229                          *
5230                          * We want to discount it the smallest amount.
5231                          * If a timer (Rack/TLP or RXT) has gone off more
5232                          * recently thats the discount we want to use (now - timer time).
5233                          * If the retransmit of the oldest packet was more recent then
5234                          * we want to use that (now - oldest-packet-last_transmit_time).
5235                          *
5236                          */
5237                         idx = rsm->r_rtr_cnt - 1;
5238                         if (TSTMP_GEQ(rack->r_ctl.rc_tlp_rxt_last_time, ((uint32_t)rsm->r_tim_lastsent[idx])))
5239                                 tstmp_touse = (uint32_t)rack->r_ctl.rc_tlp_rxt_last_time;
5240                         else
5241                                 tstmp_touse = (uint32_t)rsm->r_tim_lastsent[idx];
5242                         if (TSTMP_GT(cts, tstmp_touse))
5243                             time_since_sent = cts - tstmp_touse;
5244                 }
5245                 if (SEQ_LT(tp->snd_una, tp->snd_max) || sbavail(&(tp->t_inpcb->inp_socket->so_snd))) {
5246                         rack->r_ctl.rc_hpts_flags |= PACE_TMR_RXT;
5247                         to = tp->t_rxtcur;
5248                         if (to > time_since_sent)
5249                                 to -= time_since_sent;
5250                         else
5251                                 to = rack->r_ctl.rc_min_to;
5252                         if (to == 0)
5253                                 to = 1;
5254                         /* Special case for KEEPINIT */
5255                         if ((TCPS_HAVEESTABLISHED(tp->t_state) == 0) &&
5256                             (TP_KEEPINIT(tp) != 0) &&
5257                             rsm) {
5258                                 /*
5259                                  * We have to put a ceiling on the rxt timer
5260                                  * of the keep-init timeout.
5261                                  */
5262                                 uint32_t max_time, red;
5263
5264                                 max_time = TICKS_2_USEC(TP_KEEPINIT(tp));
5265                                 if (TSTMP_GT(cts, (uint32_t)rsm->r_tim_lastsent[0])) {
5266                                         red = (cts - (uint32_t)rsm->r_tim_lastsent[0]);
5267                                         if (red < max_time)
5268                                                 max_time -= red;
5269                                         else
5270                                                 max_time = 1;
5271                                 }
5272                                 /* Reduce timeout to the keep value if needed */
5273                                 if (max_time < to)
5274                                         to = max_time;
5275                         }
5276                         return (to);
5277                 }
5278                 return (0);
5279         }
5280         if (rsm->r_flags & RACK_ACKED) {
5281                 rsm = rack_find_lowest_rsm(rack);
5282                 if (rsm == NULL) {
5283                         /* No lowest? */
5284                         goto activate_rxt;
5285                 }
5286         }
5287         if (rack->sack_attack_disable) {
5288                 /*
5289                  * We don't want to do
5290                  * any TLP's if you are an attacker.
5291                  * Though if you are doing what
5292                  * is expected you may still have
5293                  * SACK-PASSED marks.
5294                  */
5295                 goto activate_rxt;
5296         }
5297         /* Convert from ms to usecs */
5298         if ((rsm->r_flags & RACK_SACK_PASSED) || (rsm->r_dupack >= DUP_ACK_THRESHOLD)) {
5299                 if ((tp->t_flags & TF_SENTFIN) &&
5300                     ((tp->snd_max - tp->snd_una) == 1) &&
5301                     (rsm->r_flags & RACK_HAS_FIN)) {
5302                         /*
5303                          * We don't start a rack timer if all we have is a
5304                          * FIN outstanding.
5305                          */
5306                         goto activate_rxt;
5307                 }
5308                 if ((rack->use_rack_rr == 0) &&
5309                     (IN_FASTRECOVERY(tp->t_flags)) &&
5310                     (rack->rack_no_prr == 0) &&
5311                      (rack->r_ctl.rc_prr_sndcnt  < ctf_fixed_maxseg(tp))) {
5312                         /*
5313                          * We are not cheating, in recovery  and
5314                          * not enough ack's to yet get our next
5315                          * retransmission out.
5316                          *
5317                          * Note that classified attackers do not
5318                          * get to use the rack-cheat.
5319                          */
5320                         goto activate_tlp;
5321                 }
5322                 srtt = rack_grab_rtt(tp, rack);
5323                 thresh = rack_calc_thresh_rack(rack, srtt, cts);
5324                 idx = rsm->r_rtr_cnt - 1;
5325                 exp = ((uint32_t)rsm->r_tim_lastsent[idx]) + thresh;
5326                 if (SEQ_GEQ(exp, cts)) {
5327                         to = exp - cts;
5328                         if (to < rack->r_ctl.rc_min_to) {
5329                                 to = rack->r_ctl.rc_min_to;
5330                                 if (rack->r_rr_config == 3)
5331                                         rack->rc_on_min_to = 1;
5332                         }
5333                 } else {
5334                         to = rack->r_ctl.rc_min_to;
5335                         if (rack->r_rr_config == 3)
5336                                 rack->rc_on_min_to = 1;
5337                 }
5338         } else {
5339                 /* Ok we need to do a TLP not RACK */
5340 activate_tlp:
5341                 if ((rack->rc_tlp_in_progress != 0) &&
5342                     (rack->r_ctl.rc_tlp_cnt_out >= rack_tlp_limit)) {
5343                         /*
5344                          * The previous send was a TLP and we have sent
5345                          * N TLP's without sending new data.
5346                          */
5347                         goto activate_rxt;
5348                 }
5349                 rsm = TAILQ_LAST_FAST(&rack->r_ctl.rc_tmap, rack_sendmap, r_tnext);
5350                 if (rsm == NULL) {
5351                         /* We found no rsm to TLP with. */
5352                         goto activate_rxt;
5353                 }
5354                 if (rsm->r_flags & RACK_HAS_FIN) {
5355                         /* If its a FIN we dont do TLP */
5356                         rsm = NULL;
5357                         goto activate_rxt;
5358                 }
5359                 idx = rsm->r_rtr_cnt - 1;
5360                 time_since_sent = 0;
5361                 if (TSTMP_GEQ(((uint32_t)rsm->r_tim_lastsent[idx]), rack->r_ctl.rc_tlp_rxt_last_time))
5362                         tstmp_touse = (uint32_t)rsm->r_tim_lastsent[idx];
5363                 else
5364                         tstmp_touse = (uint32_t)rack->r_ctl.rc_tlp_rxt_last_time;
5365                 if (TSTMP_GT(cts, tstmp_touse))
5366                     time_since_sent = cts - tstmp_touse;
5367                 is_tlp_timer = 1;
5368                 if (tp->t_srtt) {
5369                         if ((rack->rc_srtt_measure_made == 0) &&
5370                             (tp->t_srtt == 1)) {
5371                                 /*
5372                                  * If another stack as run and set srtt to 1,
5373                                  * then the srtt was 0, so lets use the initial.
5374                                  */
5375                                 srtt = RACK_INITIAL_RTO;
5376                         } else {
5377                                 srtt_cur = tp->t_srtt;
5378                                 srtt = srtt_cur;
5379                         }
5380                 } else
5381                         srtt = RACK_INITIAL_RTO;
5382                 /*
5383                  * If the SRTT is not keeping up and the
5384                  * rack RTT has spiked we want to use
5385                  * the last RTT not the smoothed one.
5386                  */
5387                 if (rack_tlp_use_greater &&
5388                     tp->t_srtt &&
5389                     (srtt < rack_grab_rtt(tp, rack))) {
5390                         srtt = rack_grab_rtt(tp, rack);
5391                 }
5392                 thresh = rack_calc_thresh_tlp(tp, rack, rsm, srtt);
5393                 if (thresh > time_since_sent) {
5394                         to = thresh - time_since_sent;
5395                 } else {
5396                         to = rack->r_ctl.rc_min_to;
5397                         rack_log_alt_to_to_cancel(rack,
5398                                                   thresh,               /* flex1 */
5399                                                   time_since_sent,      /* flex2 */
5400                                                   tstmp_touse,          /* flex3 */
5401                                                   rack->r_ctl.rc_tlp_rxt_last_time, /* flex4 */
5402                                                   (uint32_t)rsm->r_tim_lastsent[idx],
5403                                                   srtt,
5404                                                   idx, 99);
5405                 }
5406                 if (to < rack_tlp_min) {
5407                         to = rack_tlp_min;
5408                 }
5409                 if (to > TICKS_2_USEC(TCPTV_REXMTMAX)) {
5410                         /*
5411                          * If the TLP time works out to larger than the max
5412                          * RTO lets not do TLP.. just RTO.
5413                          */
5414                         goto activate_rxt;
5415                 }
5416         }
5417         if (is_tlp_timer == 0) {
5418                 rack->r_ctl.rc_hpts_flags |= PACE_TMR_RACK;
5419         } else {
5420                 rack->r_ctl.rc_hpts_flags |= PACE_TMR_TLP;
5421         }
5422         if (to == 0)
5423                 to = 1;
5424         return (to);
5425 }
5426
5427 static void
5428 rack_enter_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
5429 {
5430         if (rack->rc_in_persist == 0) {
5431                 if (tp->t_flags & TF_GPUTINPROG) {
5432                         /*
5433                          * Stop the goodput now, the calling of the
5434                          * measurement function clears the flag.
5435                          */
5436                         rack_do_goodput_measurement(tp, rack, tp->snd_una, __LINE__,
5437                                                     RACK_QUALITY_PERSIST);
5438                 }
5439 #ifdef NETFLIX_SHARED_CWND
5440                 if (rack->r_ctl.rc_scw) {
5441                         tcp_shared_cwnd_idle(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index);
5442                         rack->rack_scwnd_is_idle = 1;
5443                 }
5444 #endif
5445                 rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL);
5446                 if (rack->r_ctl.rc_went_idle_time == 0)
5447                         rack->r_ctl.rc_went_idle_time = 1;
5448                 rack_timer_cancel(tp, rack, cts, __LINE__);
5449                 rack->r_ctl.persist_lost_ends = 0;
5450                 rack->probe_not_answered = 0;
5451                 rack->forced_ack = 0;
5452                 tp->t_rxtshift = 0;
5453                 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
5454                               rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
5455                 rack->rc_in_persist = 1;
5456         }
5457 }
5458
5459 static void
5460 rack_exit_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
5461 {
5462         if (tcp_in_hpts(rack->rc_inp)) {
5463                 tcp_hpts_remove(rack->rc_inp);
5464                 rack->r_ctl.rc_hpts_flags = 0;
5465         }
5466 #ifdef NETFLIX_SHARED_CWND
5467         if (rack->r_ctl.rc_scw) {
5468                 tcp_shared_cwnd_active(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index);
5469                 rack->rack_scwnd_is_idle = 0;
5470         }
5471 #endif
5472         if (rack->rc_gp_dyn_mul &&
5473             (rack->use_fixed_rate == 0) &&
5474             (rack->rc_always_pace)) {
5475                 /*
5476                  * Do we count this as if a probe-rtt just
5477                  * finished?
5478                  */
5479                 uint32_t time_idle, idle_min;
5480
5481                 time_idle = tcp_get_usecs(NULL) - rack->r_ctl.rc_went_idle_time;
5482                 idle_min = rack_min_probertt_hold;
5483                 if (rack_probertt_gpsrtt_cnt_div) {
5484                         uint64_t extra;
5485                         extra = (uint64_t)rack->r_ctl.rc_gp_srtt *
5486                                 (uint64_t)rack_probertt_gpsrtt_cnt_mul;
5487                         extra /= (uint64_t)rack_probertt_gpsrtt_cnt_div;
5488                         idle_min += (uint32_t)extra;
5489                 }
5490                 if (time_idle >= idle_min) {
5491                         /* Yes, we count it as a probe-rtt. */
5492                         uint32_t us_cts;
5493
5494                         us_cts = tcp_get_usecs(NULL);
5495                         if (rack->in_probe_rtt == 0) {
5496                                 rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
5497                                 rack->r_ctl.rc_time_probertt_entered = rack->r_ctl.rc_lower_rtt_us_cts;
5498                                 rack->r_ctl.rc_time_probertt_starts = rack->r_ctl.rc_lower_rtt_us_cts;
5499                                 rack->r_ctl.rc_time_of_last_probertt = rack->r_ctl.rc_lower_rtt_us_cts;
5500                         } else {
5501                                 rack_exit_probertt(rack, us_cts);
5502                         }
5503                 }
5504         }
5505         rack->rc_in_persist = 0;
5506         rack->r_ctl.rc_went_idle_time = 0;
5507         tp->t_rxtshift = 0;
5508         RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
5509            rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
5510         rack->r_ctl.rc_agg_delayed = 0;
5511         rack->r_early = 0;
5512         rack->r_late = 0;
5513         rack->r_ctl.rc_agg_early = 0;
5514 }
5515
5516 static void
5517 rack_log_hpts_diag(struct tcp_rack *rack, uint32_t cts,
5518                    struct hpts_diag *diag, struct timeval *tv)
5519 {
5520         if (rack_verbose_logging && rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
5521                 union tcp_log_stackspecific log;
5522
5523                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
5524                 log.u_bbr.flex1 = diag->p_nxt_slot;
5525                 log.u_bbr.flex2 = diag->p_cur_slot;
5526                 log.u_bbr.flex3 = diag->slot_req;
5527                 log.u_bbr.flex4 = diag->inp_hptsslot;
5528                 log.u_bbr.flex5 = diag->slot_remaining;
5529                 log.u_bbr.flex6 = diag->need_new_to;
5530                 log.u_bbr.flex7 = diag->p_hpts_active;
5531                 log.u_bbr.flex8 = diag->p_on_min_sleep;
5532                 /* Hijack other fields as needed */
5533                 log.u_bbr.epoch = diag->have_slept;
5534                 log.u_bbr.lt_epoch = diag->yet_to_sleep;
5535                 log.u_bbr.pkts_out = diag->co_ret;
5536                 log.u_bbr.applimited = diag->hpts_sleep_time;
5537                 log.u_bbr.delivered = diag->p_prev_slot;
5538                 log.u_bbr.inflight = diag->p_runningslot;
5539                 log.u_bbr.bw_inuse = diag->wheel_slot;
5540                 log.u_bbr.rttProp = diag->wheel_cts;
5541                 log.u_bbr.timeStamp = cts;
5542                 log.u_bbr.delRate = diag->maxslots;
5543                 log.u_bbr.cur_del_rate = diag->p_curtick;
5544                 log.u_bbr.cur_del_rate <<= 32;
5545                 log.u_bbr.cur_del_rate |= diag->p_lasttick;
5546                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
5547                     &rack->rc_inp->inp_socket->so_rcv,
5548                     &rack->rc_inp->inp_socket->so_snd,
5549                     BBR_LOG_HPTSDIAG, 0,
5550                     0, &log, false, tv);
5551         }
5552
5553 }
5554
5555 static void
5556 rack_log_wakeup(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb, uint32_t len, int type)
5557 {
5558         if (rack_verbose_logging && rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
5559                 union tcp_log_stackspecific log;
5560                 struct timeval tv;
5561
5562                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
5563                 log.u_bbr.flex1 = sb->sb_flags;
5564                 log.u_bbr.flex2 = len;
5565                 log.u_bbr.flex3 = sb->sb_state;
5566                 log.u_bbr.flex8 = type;
5567                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
5568                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
5569                     &rack->rc_inp->inp_socket->so_rcv,
5570                     &rack->rc_inp->inp_socket->so_snd,
5571                     TCP_LOG_SB_WAKE, 0,
5572                     len, &log, false, &tv);
5573         }
5574 }
5575
5576 static void
5577 rack_start_hpts_timer(struct tcp_rack *rack, struct tcpcb *tp, uint32_t cts,
5578       int32_t slot, uint32_t tot_len_this_send, int sup_rack)
5579 {
5580         struct hpts_diag diag;
5581         struct inpcb *inp;
5582         struct timeval tv;
5583         uint32_t delayed_ack = 0;
5584         uint32_t hpts_timeout;
5585         uint32_t entry_slot = slot;
5586         uint8_t stopped;
5587         uint32_t left = 0;
5588         uint32_t us_cts;
5589
5590         inp = tp->t_inpcb;
5591         if ((tp->t_state == TCPS_CLOSED) ||
5592             (tp->t_state == TCPS_LISTEN)) {
5593                 return;
5594         }
5595         if (tcp_in_hpts(inp)) {
5596                 /* Already on the pacer */
5597                 return;
5598         }
5599         stopped = rack->rc_tmr_stopped;
5600         if (stopped && TSTMP_GT(rack->r_ctl.rc_timer_exp, cts)) {
5601                 left = rack->r_ctl.rc_timer_exp - cts;
5602         }
5603         rack->r_ctl.rc_timer_exp = 0;
5604         rack->r_ctl.rc_hpts_flags = 0;
5605         us_cts = tcp_get_usecs(&tv);
5606         /* Now early/late accounting */
5607         rack_log_pacing_delay_calc(rack, entry_slot, slot, 0, 0, 0, 26, __LINE__, NULL, 0);
5608         if (rack->r_early && (rack->rc_ack_can_sendout_data == 0)) {
5609                 /*
5610                  * We have a early carry over set,
5611                  * we can always add more time so we
5612                  * can always make this compensation.
5613                  *
5614                  * Note if ack's are allowed to wake us do not
5615                  * penalize the next timer for being awoke
5616                  * by an ack aka the rc_agg_early (non-paced mode).
5617                  */
5618                 slot += rack->r_ctl.rc_agg_early;
5619                 rack->r_early = 0;
5620                 rack->r_ctl.rc_agg_early = 0;
5621         }
5622         if (rack->r_late) {
5623                 /*
5624                  * This is harder, we can
5625                  * compensate some but it
5626                  * really depends on what
5627                  * the current pacing time is.
5628                  */
5629                 if (rack->r_ctl.rc_agg_delayed >= slot) {
5630                         /*
5631                          * We can't compensate for it all.
5632                          * And we have to have some time
5633                          * on the clock. We always have a min
5634                          * 10 slots (10 x 10 i.e. 100 usecs).
5635                          */
5636                         if (slot <= HPTS_TICKS_PER_SLOT) {
5637                                 /* We gain delay */
5638                                 rack->r_ctl.rc_agg_delayed += (HPTS_TICKS_PER_SLOT - slot);
5639                                 slot = HPTS_TICKS_PER_SLOT;
5640                         } else {
5641                                 /* We take off some */
5642                                 rack->r_ctl.rc_agg_delayed -= (slot - HPTS_TICKS_PER_SLOT);
5643                                 slot = HPTS_TICKS_PER_SLOT;
5644                         }
5645                 } else {
5646                         slot -= rack->r_ctl.rc_agg_delayed;
5647                         rack->r_ctl.rc_agg_delayed = 0;
5648                         /* Make sure we have 100 useconds at minimum */
5649                         if (slot < HPTS_TICKS_PER_SLOT) {
5650                                 rack->r_ctl.rc_agg_delayed = HPTS_TICKS_PER_SLOT - slot;
5651                                 slot = HPTS_TICKS_PER_SLOT;
5652                         }
5653                         if (rack->r_ctl.rc_agg_delayed == 0)
5654                                 rack->r_late = 0;
5655                 }
5656         }
5657         if (slot) {
5658                 /* We are pacing too */
5659                 rack->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;
5660         }
5661         hpts_timeout = rack_timer_start(tp, rack, cts, sup_rack);
5662 #ifdef NETFLIX_EXP_DETECTION
5663         if (rack->sack_attack_disable &&
5664             (slot < tcp_sad_pacing_interval)) {
5665                 /*
5666                  * We have a potential attacker on
5667                  * the line. We have possibly some
5668                  * (or now) pacing time set. We want to
5669                  * slow down the processing of sacks by some
5670                  * amount (if it is an attacker). Set the default
5671                  * slot for attackers in place (unless the orginal
5672                  * interval is longer). Its stored in
5673                  * micro-seconds, so lets convert to msecs.
5674                  */
5675                 slot = tcp_sad_pacing_interval;
5676         }
5677 #endif
5678         if (tp->t_flags & TF_DELACK) {
5679                 delayed_ack = TICKS_2_USEC(tcp_delacktime);
5680                 rack->r_ctl.rc_hpts_flags |= PACE_TMR_DELACK;
5681         }
5682         if (delayed_ack && ((hpts_timeout == 0) ||
5683                             (delayed_ack < hpts_timeout)))
5684                 hpts_timeout = delayed_ack;
5685         else
5686                 rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;
5687         /*
5688          * If no timers are going to run and we will fall off the hptsi
5689          * wheel, we resort to a keep-alive timer if its configured.
5690          */
5691         if ((hpts_timeout == 0) &&
5692             (slot == 0)) {
5693                 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
5694                     (tp->t_state <= TCPS_CLOSING)) {
5695                         /*
5696                          * Ok we have no timer (persists, rack, tlp, rxt  or
5697                          * del-ack), we don't have segments being paced. So
5698                          * all that is left is the keepalive timer.
5699                          */
5700                         if (TCPS_HAVEESTABLISHED(tp->t_state)) {
5701                                 /* Get the established keep-alive time */
5702                                 hpts_timeout = TICKS_2_USEC(TP_KEEPIDLE(tp));
5703                         } else {
5704                                 /*
5705                                  * Get the initial setup keep-alive time,
5706                                  * note that this is probably not going to
5707                                  * happen, since rack will be running a rxt timer
5708                                  * if a SYN of some sort is outstanding. It is
5709                                  * actually handled in rack_timeout_rxt().
5710                                  */
5711                                 hpts_timeout = TICKS_2_USEC(TP_KEEPINIT(tp));
5712                         }
5713                         rack->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP;
5714                         if (rack->in_probe_rtt) {
5715                                 /*
5716                                  * We want to instead not wake up a long time from
5717                                  * now but to wake up about the time we would
5718                                  * exit probe-rtt and initiate a keep-alive ack.
5719                                  * This will get us out of probe-rtt and update
5720                                  * our min-rtt.
5721                                  */
5722                                 hpts_timeout = rack_min_probertt_hold;
5723                         }
5724                 }
5725         }
5726         if (left && (stopped & (PACE_TMR_KEEP | PACE_TMR_DELACK)) ==
5727             (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) {
5728                 /*
5729                  * RACK, TLP, persists and RXT timers all are restartable
5730                  * based on actions input .. i.e we received a packet (ack
5731                  * or sack) and that changes things (rw, or snd_una etc).
5732                  * Thus we can restart them with a new value. For
5733                  * keep-alive, delayed_ack we keep track of what was left
5734                  * and restart the timer with a smaller value.
5735                  */
5736                 if (left < hpts_timeout)
5737                         hpts_timeout = left;
5738         }
5739         if (hpts_timeout) {
5740                 /*
5741                  * Hack alert for now we can't time-out over 2,147,483
5742                  * seconds (a bit more than 596 hours), which is probably ok
5743                  * :).
5744                  */
5745                 if (hpts_timeout > 0x7ffffffe)
5746                         hpts_timeout = 0x7ffffffe;
5747                 rack->r_ctl.rc_timer_exp = cts + hpts_timeout;
5748         }
5749         rack_log_pacing_delay_calc(rack, entry_slot, slot, hpts_timeout, 0, 0, 27, __LINE__, NULL, 0);
5750         if ((rack->gp_ready == 0) &&
5751             (rack->use_fixed_rate == 0) &&
5752             (hpts_timeout < slot) &&
5753             (rack->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) {
5754                 /*
5755                  * We have no good estimate yet for the
5756                  * old clunky burst mitigation or the
5757                  * real pacing. And the tlp or rxt is smaller
5758                  * than the pacing calculation. Lets not
5759                  * pace that long since we know the calculation
5760                  * so far is not accurate.
5761                  */
5762                 slot = hpts_timeout;
5763         }
5764         /**
5765          * Turn off all the flags for queuing by default. The
5766          * flags have important meanings to what happens when
5767          * LRO interacts with the transport. Most likely (by default now)
5768          * mbuf_queueing and ack compression are on. So the transport
5769          * has a couple of flags that control what happens (if those
5770          * are not on then these flags won't have any effect since it
5771          * won't go through the queuing LRO path).
5772          *
5773          * INP_MBUF_QUEUE_READY - This flags says that I am busy
5774          *                        pacing output, so don't disturb. But
5775          *                        it also means LRO can wake me if there
5776          *                        is a SACK arrival.
5777          *
5778          * INP_DONT_SACK_QUEUE - This flag is used in conjunction
5779          *                       with the above flag (QUEUE_READY) and
5780          *                       when present it says don't even wake me
5781          *                       if a SACK arrives.
5782          *
5783          * The idea behind these flags is that if we are pacing we
5784          * set the MBUF_QUEUE_READY and only get woken up if
5785          * a SACK arrives (which could change things) or if
5786          * our pacing timer expires. If, however, we have a rack
5787          * timer running, then we don't even want a sack to wake
5788          * us since the rack timer has to expire before we can send.
5789          *
5790          * Other cases should usually have none of the flags set
5791          * so LRO can call into us.
5792          */
5793         inp->inp_flags2 &= ~(INP_DONT_SACK_QUEUE|INP_MBUF_QUEUE_READY);
5794         if (slot) {
5795                 rack->r_ctl.rc_last_output_to = us_cts + slot;
5796                 /*
5797                  * A pacing timer (slot) is being set, in
5798                  * such a case we cannot send (we are blocked by
5799                  * the timer). So lets tell LRO that it should not
5800                  * wake us unless there is a SACK. Note this only
5801                  * will be effective if mbuf queueing is on or
5802                  * compressed acks are being processed.
5803                  */
5804                 inp->inp_flags2 |= INP_MBUF_QUEUE_READY;
5805                 /*
5806                  * But wait if we have a Rack timer running
5807                  * even a SACK should not disturb us (with
5808                  * the exception of r_rr_config 3).
5809                  */
5810                 if ((rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
5811                     (rack->r_rr_config != 3))
5812                         inp->inp_flags2 |= INP_DONT_SACK_QUEUE;
5813                 if (rack->rc_ack_can_sendout_data) {
5814                         /*
5815                          * Ahh but wait, this is that special case
5816                          * where the pacing timer can be disturbed
5817                          * backout the changes (used for non-paced
5818                          * burst limiting).
5819                          */
5820                         inp->inp_flags2 &= ~(INP_DONT_SACK_QUEUE|INP_MBUF_QUEUE_READY);
5821                 }
5822                 if ((rack->use_rack_rr) &&
5823                     (rack->r_rr_config < 2) &&
5824                     ((hpts_timeout) && (hpts_timeout < slot))) {
5825                         /*
5826                          * Arrange for the hpts to kick back in after the
5827                          * t-o if the t-o does not cause a send.
5828                          */
5829                         (void)tcp_hpts_insert_diag(tp->t_inpcb, HPTS_USEC_TO_SLOTS(hpts_timeout),
5830                                                    __LINE__, &diag);
5831                         rack_log_hpts_diag(rack, us_cts, &diag, &tv);
5832                         rack_log_to_start(rack, cts, hpts_timeout, slot, 0);
5833                 } else {
5834                         (void)tcp_hpts_insert_diag(tp->t_inpcb, HPTS_USEC_TO_SLOTS(slot),
5835                                                    __LINE__, &diag);
5836                         rack_log_hpts_diag(rack, us_cts, &diag, &tv);
5837                         rack_log_to_start(rack, cts, hpts_timeout, slot, 1);
5838                 }
5839         } else if (hpts_timeout) {
5840                 /*
5841                  * With respect to inp_flags2 here, lets let any new acks wake
5842                  * us up here. Since we are not pacing (no pacing timer), output
5843                  * can happen so we should let it. If its a Rack timer, then any inbound
5844                  * packet probably won't change the sending (we will be blocked)
5845                  * but it may change the prr stats so letting it in (the set defaults
5846                  * at the start of this block) are good enough.
5847                  */
5848                 (void)tcp_hpts_insert_diag(tp->t_inpcb, HPTS_USEC_TO_SLOTS(hpts_timeout),
5849                                            __LINE__, &diag);
5850                 rack_log_hpts_diag(rack, us_cts, &diag, &tv);
5851                 rack_log_to_start(rack, cts, hpts_timeout, slot, 0);
5852         } else {
5853                 /* No timer starting */
5854 #ifdef INVARIANTS
5855                 if (SEQ_GT(tp->snd_max, tp->snd_una)) {
5856                         panic("tp:%p rack:%p tlts:%d cts:%u slot:%u pto:%u -- no timer started?",
5857                             tp, rack, tot_len_this_send, cts, slot, hpts_timeout);
5858                 }
5859 #endif
5860         }
5861         rack->rc_tmr_stopped = 0;
5862         if (slot)
5863                 rack_log_type_bbrsnd(rack, tot_len_this_send, slot, us_cts, &tv);
5864 }
5865
5866 /*
5867  * RACK Timer, here we simply do logging and house keeping.
5868  * the normal rack_output() function will call the
5869  * appropriate thing to check if we need to do a RACK retransmit.
5870  * We return 1, saying don't proceed with rack_output only
5871  * when all timers have been stopped (destroyed PCB?).
5872  */
5873 static int
5874 rack_timeout_rack(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
5875 {
5876         /*
5877          * This timer simply provides an internal trigger to send out data.
5878          * The check_recovery_mode call will see if there are needed
5879          * retransmissions, if so we will enter fast-recovery. The output
5880          * call may or may not do the same thing depending on sysctl
5881          * settings.
5882          */
5883         struct rack_sendmap *rsm;
5884
5885         if (tp->t_timers->tt_flags & TT_STOPPED) {
5886                 return (1);
5887         }
5888         counter_u64_add(rack_to_tot, 1);
5889         if (rack->r_state && (rack->r_state != tp->t_state))
5890                 rack_set_state(tp, rack);
5891         rack->rc_on_min_to = 0;
5892         rsm = rack_check_recovery_mode(tp, cts);
5893         rack_log_to_event(rack, RACK_TO_FRM_RACK, rsm);
5894         if (rsm) {
5895                 rack->r_ctl.rc_resend = rsm;
5896                 rack->r_timer_override = 1;
5897                 if (rack->use_rack_rr) {
5898                         /*
5899                          * Don't accumulate extra pacing delay
5900                          * we are allowing the rack timer to
5901                          * over-ride pacing i.e. rrr takes precedence
5902                          * if the pacing interval is longer than the rrr
5903                          * time (in other words we get the min pacing
5904                          * time versus rrr pacing time).
5905                          */
5906                         rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
5907                 }
5908         }
5909         rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK;
5910         if (rsm == NULL) {
5911                 /* restart a timer and return 1 */
5912                 rack_start_hpts_timer(rack, tp, cts,
5913                                       0, 0, 0);
5914                 return (1);
5915         }
5916         return (0);
5917 }
5918
5919 static void
5920 rack_adjust_orig_mlen(struct rack_sendmap *rsm)
5921 {
5922         if (rsm->m->m_len > rsm->orig_m_len) {
5923                 /*
5924                  * Mbuf grew, caused by sbcompress, our offset does
5925                  * not change.
5926                  */
5927                 rsm->orig_m_len = rsm->m->m_len;
5928         } else if (rsm->m->m_len < rsm->orig_m_len) {
5929                 /*
5930                  * Mbuf shrank, trimmed off the top by an ack, our
5931                  * offset changes.
5932                  */
5933                 rsm->soff -= (rsm->orig_m_len - rsm->m->m_len);
5934                 rsm->orig_m_len = rsm->m->m_len;
5935         }
5936 }
5937
5938 static void
5939 rack_setup_offset_for_rsm(struct rack_sendmap *src_rsm, struct rack_sendmap *rsm)
5940 {
5941         struct mbuf *m;
5942         uint32_t soff;
5943
5944         if (src_rsm->m && (src_rsm->orig_m_len != src_rsm->m->m_len)) {
5945                 /* Fix up the orig_m_len and possibly the mbuf offset */
5946                 rack_adjust_orig_mlen(src_rsm);
5947         }
5948         m = src_rsm->m;
5949         soff = src_rsm->soff + (src_rsm->r_end - src_rsm->r_start);
5950         while (soff >= m->m_len) {
5951                 /* Move out past this mbuf */
5952                 soff -= m->m_len;
5953                 m = m->m_next;
5954                 KASSERT((m != NULL),
5955                         ("rsm:%p nrsm:%p hit at soff:%u null m",
5956                          src_rsm, rsm, soff));
5957         }
5958         rsm->m = m;
5959         rsm->soff = soff;
5960         rsm->orig_m_len = m->m_len;
5961 }
5962
5963 static __inline void
5964 rack_clone_rsm(struct tcp_rack *rack, struct rack_sendmap *nrsm,
5965                struct rack_sendmap *rsm, uint32_t start)
5966 {
5967         int idx;
5968
5969         nrsm->r_start = start;
5970         nrsm->r_end = rsm->r_end;
5971         nrsm->r_rtr_cnt = rsm->r_rtr_cnt;
5972         nrsm->r_flags = rsm->r_flags;
5973         nrsm->r_dupack = rsm->r_dupack;
5974         nrsm->r_no_rtt_allowed = rsm->r_no_rtt_allowed;
5975         nrsm->r_rtr_bytes = 0;
5976         nrsm->r_fas = rsm->r_fas;
5977         rsm->r_end = nrsm->r_start;
5978         nrsm->r_just_ret = rsm->r_just_ret;
5979         for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) {
5980                 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx];
5981         }
5982         /* Now if we have SYN flag we keep it on the left edge */
5983         if (nrsm->r_flags & RACK_HAS_SYN)
5984                 nrsm->r_flags &= ~RACK_HAS_SYN;
5985         /* Now if we have a FIN flag we keep it on the right edge */
5986         if (rsm->r_flags & RACK_HAS_FIN)
5987                 rsm->r_flags &= ~RACK_HAS_FIN;
5988         /* Push bit must go to the right edge as well */
5989         if (rsm->r_flags & RACK_HAD_PUSH)
5990                 rsm->r_flags &= ~RACK_HAD_PUSH;
5991         /* Clone over the state of the hw_tls flag */
5992         nrsm->r_hw_tls = rsm->r_hw_tls;
5993         /*
5994          * Now we need to find nrsm's new location in the mbuf chain
5995          * we basically calculate a new offset, which is soff +
5996          * how much is left in original rsm. Then we walk out the mbuf
5997          * chain to find the righ position, it may be the same mbuf
5998          * or maybe not.
5999          */
6000         KASSERT(((rsm->m != NULL) ||
6001                  (rsm->r_flags & (RACK_HAS_SYN|RACK_HAS_FIN))),
6002                 ("rsm:%p nrsm:%p rack:%p -- rsm->m is NULL?", rsm, nrsm, rack));
6003         if (rsm->m)
6004                 rack_setup_offset_for_rsm(rsm, nrsm);
6005 }
6006
6007 static struct rack_sendmap *
6008 rack_merge_rsm(struct tcp_rack *rack,
6009                struct rack_sendmap *l_rsm,
6010                struct rack_sendmap *r_rsm)
6011 {
6012         /*
6013          * We are merging two ack'd RSM's,
6014          * the l_rsm is on the left (lower seq
6015          * values) and the r_rsm is on the right
6016          * (higher seq value). The simplest way
6017          * to merge these is to move the right
6018          * one into the left. I don't think there
6019          * is any reason we need to try to find
6020          * the oldest (or last oldest retransmitted).
6021          */
6022 #ifdef INVARIANTS
6023         struct rack_sendmap *rm;
6024 #endif
6025         rack_log_map_chg(rack->rc_tp, rack, NULL,
6026                          l_rsm, r_rsm, MAP_MERGE, r_rsm->r_end, __LINE__);
6027         l_rsm->r_end = r_rsm->r_end;
6028         if (l_rsm->r_dupack < r_rsm->r_dupack)
6029                 l_rsm->r_dupack = r_rsm->r_dupack;
6030         if (r_rsm->r_rtr_bytes)
6031                 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes;
6032         if (r_rsm->r_in_tmap) {
6033                 /* This really should not happen */
6034                 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, r_rsm, r_tnext);
6035                 r_rsm->r_in_tmap = 0;
6036         }
6037
6038         /* Now the flags */
6039         if (r_rsm->r_flags & RACK_HAS_FIN)
6040                 l_rsm->r_flags |= RACK_HAS_FIN;
6041         if (r_rsm->r_flags & RACK_TLP)
6042                 l_rsm->r_flags |= RACK_TLP;
6043         if (r_rsm->r_flags & RACK_RWND_COLLAPSED)
6044                 l_rsm->r_flags |= RACK_RWND_COLLAPSED;
6045         if ((r_rsm->r_flags & RACK_APP_LIMITED)  &&
6046             ((l_rsm->r_flags & RACK_APP_LIMITED) == 0)) {
6047                 /*
6048                  * If both are app-limited then let the
6049                  * free lower the count. If right is app
6050                  * limited and left is not, transfer.
6051                  */
6052                 l_rsm->r_flags |= RACK_APP_LIMITED;
6053                 r_rsm->r_flags &= ~RACK_APP_LIMITED;
6054                 if (r_rsm == rack->r_ctl.rc_first_appl)
6055                         rack->r_ctl.rc_first_appl = l_rsm;
6056         }
6057 #ifndef INVARIANTS
6058         (void)RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, r_rsm);
6059 #else
6060         rm = RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, r_rsm);
6061         if (rm != r_rsm) {
6062                 panic("removing head in rack:%p rsm:%p rm:%p",
6063                       rack, r_rsm, rm);
6064         }
6065 #endif
6066         if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) {
6067                 /* Transfer the split limit to the map we free */
6068                 r_rsm->r_limit_type = l_rsm->r_limit_type;
6069                 l_rsm->r_limit_type = 0;
6070         }
6071         rack_free(rack, r_rsm);
6072         return (l_rsm);
6073 }
6074
6075 /*
6076  * TLP Timer, here we simply setup what segment we want to
6077  * have the TLP expire on, the normal rack_output() will then
6078  * send it out.
6079  *
6080  * We return 1, saying don't proceed with rack_output only
6081  * when all timers have been stopped (destroyed PCB?).
6082  */
6083 static int
6084 rack_timeout_tlp(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, uint8_t *doing_tlp)
6085 {
6086         /*
6087          * Tail Loss Probe.
6088          */
6089         struct rack_sendmap *rsm = NULL;
6090 #ifdef INVARIANTS
6091         struct rack_sendmap *insret;
6092 #endif
6093         struct socket *so;
6094         uint32_t amm;
6095         uint32_t out, avail;
6096         int collapsed_win = 0;
6097
6098         if (tp->t_timers->tt_flags & TT_STOPPED) {
6099                 return (1);
6100         }
6101         if (TSTMP_LT(cts, rack->r_ctl.rc_timer_exp)) {
6102                 /* Its not time yet */
6103                 return (0);
6104         }
6105         if (ctf_progress_timeout_check(tp, true)) {
6106                 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__);
6107                 return (-ETIMEDOUT);    /* tcp_drop() */
6108         }
6109         /*
6110          * A TLP timer has expired. We have been idle for 2 rtts. So we now
6111          * need to figure out how to force a full MSS segment out.
6112          */
6113         rack_log_to_event(rack, RACK_TO_FRM_TLP, NULL);
6114         rack->r_ctl.retran_during_recovery = 0;
6115         rack->r_ctl.dsack_byte_cnt = 0;
6116         counter_u64_add(rack_tlp_tot, 1);
6117         if (rack->r_state && (rack->r_state != tp->t_state))
6118                 rack_set_state(tp, rack);
6119         so = tp->t_inpcb->inp_socket;
6120         avail = sbavail(&so->so_snd);
6121         out = tp->snd_max - tp->snd_una;
6122         if ((out > tp->snd_wnd) || rack->rc_has_collapsed) {
6123                 /* special case, we need a retransmission */
6124                 collapsed_win = 1;
6125                 goto need_retran;
6126         }
6127         if (rack->r_ctl.dsack_persist && (rack->r_ctl.rc_tlp_cnt_out >= 1)) {
6128                 rack->r_ctl.dsack_persist--;
6129                 if (rack->r_ctl.num_dsack && (rack->r_ctl.dsack_persist == 0)) {
6130                         rack->r_ctl.num_dsack = 0;
6131                 }
6132                 rack_log_dsack_event(rack, 1, __LINE__, 0, 0);
6133         }
6134         if ((tp->t_flags & TF_GPUTINPROG) &&
6135             (rack->r_ctl.rc_tlp_cnt_out == 1)) {
6136                 /*
6137                  * If this is the second in a row
6138                  * TLP and we are doing a measurement
6139                  * its time to abandon the measurement.
6140                  * Something is likely broken on
6141                  * the clients network and measuring a
6142                  * broken network does us no good.
6143                  */
6144                 tp->t_flags &= ~TF_GPUTINPROG;
6145                 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
6146                                            rack->r_ctl.rc_gp_srtt /*flex1*/,
6147                                            tp->gput_seq,
6148                                            0, 0, 18, __LINE__, NULL, 0);
6149         }
6150         /*
6151          * Check our send oldest always settings, and if
6152          * there is an oldest to send jump to the need_retran.
6153          */
6154         if (rack_always_send_oldest && (TAILQ_EMPTY(&rack->r_ctl.rc_tmap) == 0))
6155                 goto need_retran;
6156
6157         if (avail > out) {
6158                 /* New data is available */
6159                 amm = avail - out;
6160                 if (amm > ctf_fixed_maxseg(tp)) {
6161                         amm = ctf_fixed_maxseg(tp);
6162                         if ((amm + out) > tp->snd_wnd) {
6163                                 /* We are rwnd limited */
6164                                 goto need_retran;
6165                         }
6166                 } else if (amm < ctf_fixed_maxseg(tp)) {
6167                         /* not enough to fill a MTU */
6168                         goto need_retran;
6169                 }
6170                 if (IN_FASTRECOVERY(tp->t_flags)) {
6171                         /* Unlikely */
6172                         if (rack->rack_no_prr == 0) {
6173                                 if (out + amm <= tp->snd_wnd) {
6174                                         rack->r_ctl.rc_prr_sndcnt = amm;
6175                                         rack->r_ctl.rc_tlp_new_data = amm;
6176                                         rack_log_to_prr(rack, 4, 0, __LINE__);
6177                                 }
6178                         } else
6179                                 goto need_retran;
6180                 } else {
6181                         /* Set the send-new override */
6182                         if (out + amm <= tp->snd_wnd)
6183                                 rack->r_ctl.rc_tlp_new_data = amm;
6184                         else
6185                                 goto need_retran;
6186                 }
6187                 rack->r_ctl.rc_tlpsend = NULL;
6188                 counter_u64_add(rack_tlp_newdata, 1);
6189                 goto send;
6190         }
6191 need_retran:
6192         /*
6193          * Ok we need to arrange the last un-acked segment to be re-sent, or
6194          * optionally the first un-acked segment.
6195          */
6196         if (collapsed_win == 0) {
6197                 if (rack_always_send_oldest)
6198                         rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
6199                 else {
6200                         rsm = RB_MAX(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
6201                         if (rsm && (rsm->r_flags & (RACK_ACKED | RACK_HAS_FIN))) {
6202                                 rsm = rack_find_high_nonack(rack, rsm);
6203                         }
6204                 }
6205                 if (rsm == NULL) {
6206 #ifdef TCP_BLACKBOX
6207                         tcp_log_dump_tp_logbuf(tp, "nada counter trips", M_NOWAIT, true);
6208 #endif
6209                         goto out;
6210                 }
6211         } else {
6212                 /*
6213                  * We must find the last segment
6214                  * that was acceptable by the client.
6215                  */
6216                 RB_FOREACH_REVERSE(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) {
6217                         if ((rsm->r_flags & RACK_RWND_COLLAPSED) == 0) {
6218                                 /* Found one */
6219                                 break;
6220                         }
6221                 }
6222                 if (rsm == NULL) {
6223                         /* None? if so send the first */
6224                         rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
6225                         if (rsm == NULL) {
6226 #ifdef TCP_BLACKBOX
6227                                 tcp_log_dump_tp_logbuf(tp, "nada counter trips", M_NOWAIT, true);
6228 #endif
6229                                 goto out;
6230                         }
6231                 }
6232         }
6233         if ((rsm->r_end - rsm->r_start) > ctf_fixed_maxseg(tp)) {
6234                 /*
6235                  * We need to split this the last segment in two.
6236                  */
6237                 struct rack_sendmap *nrsm;
6238
6239                 nrsm = rack_alloc_full_limit(rack);
6240                 if (nrsm == NULL) {
6241                         /*
6242                          * No memory to split, we will just exit and punt
6243                          * off to the RXT timer.
6244                          */
6245                         goto out;
6246                 }
6247                 rack_clone_rsm(rack, nrsm, rsm,
6248                                (rsm->r_end - ctf_fixed_maxseg(tp)));
6249                 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 0, __LINE__);
6250 #ifndef INVARIANTS
6251                 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
6252 #else
6253                 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
6254                 if (insret != NULL) {
6255                         panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p",
6256                               nrsm, insret, rack, rsm);
6257                 }
6258 #endif
6259                 if (rsm->r_in_tmap) {
6260                         TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
6261                         nrsm->r_in_tmap = 1;
6262                 }
6263                 rsm = nrsm;
6264         }
6265         rack->r_ctl.rc_tlpsend = rsm;
6266 send:
6267         /* Make sure output path knows we are doing a TLP */
6268         *doing_tlp = 1;
6269         rack->r_timer_override = 1;
6270         rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;
6271         return (0);
6272 out:
6273         rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;
6274         return (0);
6275 }
6276
6277 /*
6278  * Delayed ack Timer, here we simply need to setup the
6279  * ACK_NOW flag and remove the DELACK flag. From there
6280  * the output routine will send the ack out.
6281  *
6282  * We only return 1, saying don't proceed, if all timers
6283  * are stopped (destroyed PCB?).
6284  */
6285 static int
6286 rack_timeout_delack(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
6287 {
6288         if (tp->t_timers->tt_flags & TT_STOPPED) {
6289                 return (1);
6290         }
6291         rack_log_to_event(rack, RACK_TO_FRM_DELACK, NULL);
6292         tp->t_flags &= ~TF_DELACK;
6293         tp->t_flags |= TF_ACKNOW;
6294         KMOD_TCPSTAT_INC(tcps_delack);
6295         rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;
6296         return (0);
6297 }
6298
6299 /*
6300  * Persists timer, here we simply send the
6301  * same thing as a keepalive will.
6302  * the one byte send.
6303  *
6304  * We only return 1, saying don't proceed, if all timers
6305  * are stopped (destroyed PCB?).
6306  */
6307 static int
6308 rack_timeout_persist(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
6309 {
6310         struct tcptemp *t_template;
6311 #ifdef INVARIANTS
6312         struct inpcb *inp = tp->t_inpcb;
6313 #endif
6314         int32_t retval = 1;
6315
6316         if (tp->t_timers->tt_flags & TT_STOPPED) {
6317                 return (1);
6318         }
6319         if (rack->rc_in_persist == 0)
6320                 return (0);
6321         if (ctf_progress_timeout_check(tp, false)) {
6322                 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
6323                 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__);
6324                 counter_u64_add(rack_persists_lost_ends, rack->r_ctl.persist_lost_ends);
6325                 return (-ETIMEDOUT);    /* tcp_drop() */
6326         }
6327         KASSERT(inp != NULL, ("%s: tp %p tp->t_inpcb == NULL", __func__, tp));
6328         /*
6329          * Persistence timer into zero window. Force a byte to be output, if
6330          * possible.
6331          */
6332         KMOD_TCPSTAT_INC(tcps_persisttimeo);
6333         /*
6334          * Hack: if the peer is dead/unreachable, we do not time out if the
6335          * window is closed.  After a full backoff, drop the connection if
6336          * the idle time (no responses to probes) reaches the maximum
6337          * backoff that we would use if retransmitting.
6338          */
6339         if (tp->t_rxtshift == TCP_MAXRXTSHIFT &&
6340             (ticks - tp->t_rcvtime >= tcp_maxpersistidle ||
6341              TICKS_2_USEC(ticks - tp->t_rcvtime) >= RACK_REXMTVAL(tp) * tcp_totbackoff)) {
6342                 KMOD_TCPSTAT_INC(tcps_persistdrop);
6343                 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
6344                 counter_u64_add(rack_persists_lost_ends, rack->r_ctl.persist_lost_ends);
6345                 retval = -ETIMEDOUT;    /* tcp_drop() */
6346                 goto out;
6347         }
6348         if ((sbavail(&rack->rc_inp->inp_socket->so_snd) == 0) &&
6349             tp->snd_una == tp->snd_max)
6350                 rack_exit_persist(tp, rack, cts);
6351         rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT;
6352         /*
6353          * If the user has closed the socket then drop a persisting
6354          * connection after a much reduced timeout.
6355          */
6356         if (tp->t_state > TCPS_CLOSE_WAIT &&
6357             (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) {
6358                 KMOD_TCPSTAT_INC(tcps_persistdrop);
6359                 tcp_log_end_status(tp, TCP_EI_STATUS_PERSIST_MAX);
6360                 counter_u64_add(rack_persists_lost_ends, rack->r_ctl.persist_lost_ends);
6361                 retval = -ETIMEDOUT;    /* tcp_drop() */
6362                 goto out;
6363         }
6364         t_template = tcpip_maketemplate(rack->rc_inp);
6365         if (t_template) {
6366                 /* only set it if we were answered */
6367                 if (rack->forced_ack == 0) {
6368                         rack->forced_ack = 1;
6369                         rack->r_ctl.forced_ack_ts = tcp_get_usecs(NULL);
6370                 } else {
6371                         rack->probe_not_answered = 1;
6372                         counter_u64_add(rack_persists_loss, 1);
6373                         rack->r_ctl.persist_lost_ends++;
6374                 }
6375                 counter_u64_add(rack_persists_sends, 1);
6376                 tcp_respond(tp, t_template->tt_ipgen,
6377                             &t_template->tt_t, (struct mbuf *)NULL,
6378                             tp->rcv_nxt, tp->snd_una - 1, 0);
6379                 /* This sends an ack */
6380                 if (tp->t_flags & TF_DELACK)
6381                         tp->t_flags &= ~TF_DELACK;
6382                 free(t_template, M_TEMP);
6383         }
6384         if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
6385                 tp->t_rxtshift++;
6386 out:
6387         rack_log_to_event(rack, RACK_TO_FRM_PERSIST, NULL);
6388         rack_start_hpts_timer(rack, tp, cts,
6389                               0, 0, 0);
6390         return (retval);
6391 }
6392
6393 /*
6394  * If a keepalive goes off, we had no other timers
6395  * happening. We always return 1 here since this
6396  * routine either drops the connection or sends
6397  * out a segment with respond.
6398  */
6399 static int
6400 rack_timeout_keepalive(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
6401 {
6402         struct tcptemp *t_template;
6403         struct inpcb *inp;
6404
6405         if (tp->t_timers->tt_flags & TT_STOPPED) {
6406                 return (1);
6407         }
6408         rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP;
6409         inp = tp->t_inpcb;
6410         rack_log_to_event(rack, RACK_TO_FRM_KEEP, NULL);
6411         /*
6412          * Keep-alive timer went off; send something or drop connection if
6413          * idle for too long.
6414          */
6415         KMOD_TCPSTAT_INC(tcps_keeptimeo);
6416         if (tp->t_state < TCPS_ESTABLISHED)
6417                 goto dropit;
6418         if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
6419             tp->t_state <= TCPS_CLOSING) {
6420                 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp))
6421                         goto dropit;
6422                 /*
6423                  * Send a packet designed to force a response if the peer is
6424                  * up and reachable: either an ACK if the connection is
6425                  * still alive, or an RST if the peer has closed the
6426                  * connection due to timeout or reboot. Using sequence
6427                  * number tp->snd_una-1 causes the transmitted zero-length
6428                  * segment to lie outside the receive window; by the
6429                  * protocol spec, this requires the correspondent TCP to
6430                  * respond.
6431                  */
6432                 KMOD_TCPSTAT_INC(tcps_keepprobe);
6433                 t_template = tcpip_maketemplate(inp);
6434                 if (t_template) {
6435                         if (rack->forced_ack == 0) {
6436                                 rack->forced_ack = 1;
6437                                 rack->r_ctl.forced_ack_ts = tcp_get_usecs(NULL);
6438                         } else {
6439                                 rack->probe_not_answered = 1;
6440                         }
6441                         tcp_respond(tp, t_template->tt_ipgen,
6442                             &t_template->tt_t, (struct mbuf *)NULL,
6443                             tp->rcv_nxt, tp->snd_una - 1, 0);
6444                         free(t_template, M_TEMP);
6445                 }
6446         }
6447         rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
6448         return (1);
6449 dropit:
6450         KMOD_TCPSTAT_INC(tcps_keepdrops);
6451         tcp_log_end_status(tp, TCP_EI_STATUS_KEEP_MAX);
6452         return (-ETIMEDOUT);    /* tcp_drop() */
6453 }
6454
6455 /*
6456  * Retransmit helper function, clear up all the ack
6457  * flags and take care of important book keeping.
6458  */
6459 static void
6460 rack_remxt_tmr(struct tcpcb *tp)
6461 {
6462         /*
6463          * The retransmit timer went off, all sack'd blocks must be
6464          * un-acked.
6465          */
6466         struct rack_sendmap *rsm, *trsm = NULL;
6467         struct tcp_rack *rack;
6468
6469         rack = (struct tcp_rack *)tp->t_fb_ptr;
6470         rack_timer_cancel(tp, rack, tcp_get_usecs(NULL), __LINE__);
6471         rack_log_to_event(rack, RACK_TO_FRM_TMR, NULL);
6472         if (rack->r_state && (rack->r_state != tp->t_state))
6473                 rack_set_state(tp, rack);
6474         /*
6475          * Ideally we would like to be able to
6476          * mark SACK-PASS on anything not acked here.
6477          *
6478          * However, if we do that we would burst out
6479          * all that data 1ms apart. This would be unwise,
6480          * so for now we will just let the normal rxt timer
6481          * and tlp timer take care of it.
6482          *
6483          * Also we really need to stick them back in sequence
6484          * order. This way we send in the proper order and any
6485          * sacks that come floating in will "re-ack" the data.
6486          * To do this we zap the tmap with an INIT and then
6487          * walk through and place every rsm in the RB tree
6488          * back in its seq ordered place.
6489          */
6490         TAILQ_INIT(&rack->r_ctl.rc_tmap);
6491         RB_FOREACH(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) {
6492                 rsm->r_dupack = 0;
6493                 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
6494                 /* We must re-add it back to the tlist */
6495                 if (trsm == NULL) {
6496                         TAILQ_INSERT_HEAD(&rack->r_ctl.rc_tmap, rsm, r_tnext);
6497                 } else {
6498                         TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, trsm, rsm, r_tnext);
6499                 }
6500                 rsm->r_in_tmap = 1;
6501                 trsm = rsm;
6502                 if (rsm->r_flags & RACK_ACKED)
6503                         rsm->r_flags |= RACK_WAS_ACKED;
6504                 rsm->r_flags &= ~(RACK_ACKED | RACK_SACK_PASSED | RACK_WAS_SACKPASS);
6505                 rsm->r_flags |= RACK_MUST_RXT;
6506         }
6507         /* Clear the count (we just un-acked them) */
6508         rack->r_ctl.rc_last_timeout_snduna = tp->snd_una;
6509         rack->r_ctl.rc_sacked = 0;
6510         rack->r_ctl.rc_sacklast = NULL;
6511         rack->r_ctl.rc_agg_delayed = 0;
6512         rack->r_early = 0;
6513         rack->r_ctl.rc_agg_early = 0;
6514         rack->r_late = 0;
6515         /* Clear the tlp rtx mark */
6516         rack->r_ctl.rc_resend = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
6517         if (rack->r_ctl.rc_resend != NULL)
6518                 rack->r_ctl.rc_resend->r_flags |= RACK_TO_REXT;
6519         rack->r_ctl.rc_prr_sndcnt = 0;
6520         rack_log_to_prr(rack, 6, 0, __LINE__);
6521         rack->r_timer_override = 1;
6522         if ((((tp->t_flags & TF_SACK_PERMIT) == 0)
6523 #ifdef NETFLIX_EXP_DETECTION
6524             || (rack->sack_attack_disable != 0)
6525 #endif
6526                     ) && ((tp->t_flags & TF_SENTFIN) == 0)) {
6527                 /*
6528                  * For non-sack customers new data
6529                  * needs to go out as retransmits until
6530                  * we retransmit up to snd_max.
6531                  */
6532                 rack->r_must_retran = 1;
6533                 rack->r_ctl.rc_out_at_rto = ctf_flight_size(rack->rc_tp,
6534                                                 rack->r_ctl.rc_sacked);
6535         }
6536         rack->r_ctl.rc_snd_max_at_rto = tp->snd_max;
6537 }
6538
6539 static void
6540 rack_convert_rtts(struct tcpcb *tp)
6541 {
6542         if (tp->t_srtt > 1) {
6543                 uint32_t val, frac;
6544
6545                 val = tp->t_srtt >> TCP_RTT_SHIFT;
6546                 frac = tp->t_srtt & 0x1f;
6547                 tp->t_srtt = TICKS_2_USEC(val);
6548                 /*
6549                  * frac is the fractional part of the srtt (if any)
6550                  * but its in ticks and every bit represents
6551                  * 1/32nd of a hz.
6552                  */
6553                 if (frac) {
6554                         if (hz == 1000) {
6555                                 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
6556                         } else {
6557                                 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
6558                         }
6559                         tp->t_srtt += frac;
6560                 }
6561         }
6562         if (tp->t_rttvar) {
6563                 uint32_t val, frac;
6564
6565                 val = tp->t_rttvar >> TCP_RTTVAR_SHIFT;
6566                 frac = tp->t_rttvar & 0x1f;
6567                 tp->t_rttvar = TICKS_2_USEC(val);
6568                 /*
6569                  * frac is the fractional part of the srtt (if any)
6570                  * but its in ticks and every bit represents
6571                  * 1/32nd of a hz.
6572                  */
6573                 if (frac) {
6574                         if (hz == 1000) {
6575                                 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_MSEC) / (uint64_t)TCP_RTT_SCALE);
6576                         } else {
6577                                 frac = (((uint64_t)frac * (uint64_t)HPTS_USEC_IN_SEC) / ((uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE));
6578                         }
6579                         tp->t_rttvar += frac;
6580                 }
6581         }
6582         tp->t_rxtcur = RACK_REXMTVAL(tp);
6583         if (TCPS_HAVEESTABLISHED(tp->t_state)) {
6584                 tp->t_rxtcur += TICKS_2_USEC(tcp_rexmit_slop);
6585         }
6586         if (tp->t_rxtcur > rack_rto_max) {
6587                 tp->t_rxtcur = rack_rto_max;
6588         }
6589 }
6590
6591 static void
6592 rack_cc_conn_init(struct tcpcb *tp)
6593 {
6594         struct tcp_rack *rack;
6595         uint32_t srtt;
6596
6597         rack = (struct tcp_rack *)tp->t_fb_ptr;
6598         srtt = tp->t_srtt;
6599         cc_conn_init(tp);
6600         /*
6601          * Now convert to rack's internal format,
6602          * if required.
6603          */
6604         if ((srtt == 0) && (tp->t_srtt != 0))
6605                 rack_convert_rtts(tp);
6606         /*
6607          * We want a chance to stay in slowstart as
6608          * we create a connection. TCP spec says that
6609          * initially ssthresh is infinite. For our
6610          * purposes that is the snd_wnd.
6611          */
6612         if (tp->snd_ssthresh < tp->snd_wnd) {
6613                 tp->snd_ssthresh = tp->snd_wnd;
6614         }
6615         /*
6616          * We also want to assure a IW worth of
6617          * data can get inflight.
6618          */
6619         if (rc_init_window(rack) < tp->snd_cwnd)
6620                 tp->snd_cwnd = rc_init_window(rack);
6621 }
6622
6623 /*
6624  * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
6625  * we will setup to retransmit the lowest seq number outstanding.
6626  */
6627 static int
6628 rack_timeout_rxt(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts)
6629 {
6630         int32_t rexmt;
6631         int32_t retval = 0;
6632         bool isipv6;
6633
6634         if (tp->t_timers->tt_flags & TT_STOPPED) {
6635                 return (1);
6636         }
6637         if ((tp->t_flags & TF_GPUTINPROG) &&
6638             (tp->t_rxtshift)) {
6639                 /*
6640                  * We have had a second timeout
6641                  * measurements on successive rxt's are not profitable.
6642                  * It is unlikely to be of any use (the network is
6643                  * broken or the client went away).
6644                  */
6645                 tp->t_flags &= ~TF_GPUTINPROG;
6646                 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
6647                                            rack->r_ctl.rc_gp_srtt /*flex1*/,
6648                                            tp->gput_seq,
6649                                            0, 0, 18, __LINE__, NULL, 0);
6650         }
6651         if (ctf_progress_timeout_check(tp, false)) {
6652                 tcp_log_end_status(tp, TCP_EI_STATUS_RETRAN);
6653                 rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__);
6654                 return (-ETIMEDOUT);    /* tcp_drop() */
6655         }
6656         rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT;
6657         rack->r_ctl.retran_during_recovery = 0;
6658         rack->r_ctl.dsack_byte_cnt = 0;
6659         if (IN_FASTRECOVERY(tp->t_flags))
6660                 tp->t_flags |= TF_WASFRECOVERY;
6661         else
6662                 tp->t_flags &= ~TF_WASFRECOVERY;
6663         if (IN_CONGRECOVERY(tp->t_flags))
6664                 tp->t_flags |= TF_WASCRECOVERY;
6665         else
6666                 tp->t_flags &= ~TF_WASCRECOVERY;
6667         if (TCPS_HAVEESTABLISHED(tp->t_state) &&
6668             (tp->snd_una == tp->snd_max)) {
6669                 /* Nothing outstanding .. nothing to do */
6670                 return (0);
6671         }
6672         if (rack->r_ctl.dsack_persist) {
6673                 rack->r_ctl.dsack_persist--;
6674                 if (rack->r_ctl.num_dsack && (rack->r_ctl.dsack_persist == 0)) {
6675                         rack->r_ctl.num_dsack = 0;
6676                 }
6677                 rack_log_dsack_event(rack, 1, __LINE__, 0, 0);
6678         }
6679         /*
6680          * Rack can only run one timer  at a time, so we cannot
6681          * run a KEEPINIT (gating SYN sending) and a retransmit
6682          * timer for the SYN. So if we are in a front state and
6683          * have a KEEPINIT timer we need to check the first transmit
6684          * against now to see if we have exceeded the KEEPINIT time
6685          * (if one is set).
6686          */
6687         if ((TCPS_HAVEESTABLISHED(tp->t_state) == 0) &&
6688             (TP_KEEPINIT(tp) != 0)) {
6689                 struct rack_sendmap *rsm;
6690
6691                 rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
6692                 if (rsm) {
6693                         /* Ok we have something outstanding to test keepinit with */
6694                         if ((TSTMP_GT(cts, (uint32_t)rsm->r_tim_lastsent[0])) &&
6695                             ((cts - (uint32_t)rsm->r_tim_lastsent[0]) >= TICKS_2_USEC(TP_KEEPINIT(tp)))) {
6696                                 /* We have exceeded the KEEPINIT time */
6697                                 tcp_log_end_status(tp, TCP_EI_STATUS_KEEP_MAX);
6698                                 goto drop_it;
6699                         }
6700                 }
6701         }
6702         /*
6703          * Retransmission timer went off.  Message has not been acked within
6704          * retransmit interval.  Back off to a longer retransmit interval
6705          * and retransmit one segment.
6706          */
6707         rack_remxt_tmr(tp);
6708         if ((rack->r_ctl.rc_resend == NULL) ||
6709             ((rack->r_ctl.rc_resend->r_flags & RACK_RWND_COLLAPSED) == 0)) {
6710                 /*
6711                  * If the rwnd collapsed on
6712                  * the one we are retransmitting
6713                  * it does not count against the
6714                  * rxt count.
6715                  */
6716                 tp->t_rxtshift++;
6717         }
6718         if (tp->t_rxtshift > TCP_MAXRXTSHIFT) {
6719                 tcp_log_end_status(tp, TCP_EI_STATUS_RETRAN);
6720 drop_it:
6721                 tp->t_rxtshift = TCP_MAXRXTSHIFT;
6722                 KMOD_TCPSTAT_INC(tcps_timeoutdrop);
6723                 /* XXXGL: previously t_softerror was casted to uint16_t */
6724                 MPASS(tp->t_softerror >= 0);
6725                 retval = tp->t_softerror ? -tp->t_softerror : -ETIMEDOUT;
6726                 goto out;       /* tcp_drop() */
6727         }
6728         if (tp->t_state == TCPS_SYN_SENT) {
6729                 /*
6730                  * If the SYN was retransmitted, indicate CWND to be limited
6731                  * to 1 segment in cc_conn_init().
6732                  */
6733                 tp->snd_cwnd = 1;
6734         } else if (tp->t_rxtshift == 1) {
6735                 /*
6736                  * first retransmit; record ssthresh and cwnd so they can be
6737                  * recovered if this turns out to be a "bad" retransmit. A
6738                  * retransmit is considered "bad" if an ACK for this segment
6739                  * is received within RTT/2 interval; the assumption here is
6740                  * that the ACK was already in flight.  See "On Estimating
6741                  * End-to-End Network Path Properties" by Allman and Paxson
6742                  * for more details.
6743                  */
6744                 tp->snd_cwnd_prev = tp->snd_cwnd;
6745                 tp->snd_ssthresh_prev = tp->snd_ssthresh;
6746                 tp->snd_recover_prev = tp->snd_recover;
6747                 tp->t_badrxtwin = ticks + (USEC_2_TICKS(tp->t_srtt)/2);
6748                 tp->t_flags |= TF_PREVVALID;
6749         } else if ((tp->t_flags & TF_RCVD_TSTMP) == 0)
6750                 tp->t_flags &= ~TF_PREVVALID;
6751         KMOD_TCPSTAT_INC(tcps_rexmttimeo);
6752         if ((tp->t_state == TCPS_SYN_SENT) ||
6753             (tp->t_state == TCPS_SYN_RECEIVED))
6754                 rexmt = RACK_INITIAL_RTO * tcp_backoff[tp->t_rxtshift];
6755         else
6756                 rexmt = max(rack_rto_min, (tp->t_srtt + (tp->t_rttvar << 2))) * tcp_backoff[tp->t_rxtshift];
6757
6758         RACK_TCPT_RANGESET(tp->t_rxtcur, rexmt,
6759            max(rack_rto_min, rexmt), rack_rto_max, rack->r_ctl.timer_slop);
6760         /*
6761          * We enter the path for PLMTUD if connection is established or, if
6762          * connection is FIN_WAIT_1 status, reason for the last is that if
6763          * amount of data we send is very small, we could send it in couple
6764          * of packets and process straight to FIN. In that case we won't
6765          * catch ESTABLISHED state.
6766          */
6767 #ifdef INET6
6768         isipv6 = (tp->t_inpcb->inp_vflag & INP_IPV6) ? true : false;
6769 #else
6770         isipv6 = false;
6771 #endif
6772         if (((V_tcp_pmtud_blackhole_detect == 1) ||
6773             (V_tcp_pmtud_blackhole_detect == 2 && !isipv6) ||
6774             (V_tcp_pmtud_blackhole_detect == 3 && isipv6)) &&
6775             ((tp->t_state == TCPS_ESTABLISHED) ||
6776             (tp->t_state == TCPS_FIN_WAIT_1))) {
6777                 /*
6778                  * Idea here is that at each stage of mtu probe (usually,
6779                  * 1448 -> 1188 -> 524) should be given 2 chances to recover
6780                  * before further clamping down. 'tp->t_rxtshift % 2 == 0'
6781                  * should take care of that.
6782                  */
6783                 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) ==
6784                     (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) &&
6785                     (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&
6786                     tp->t_rxtshift % 2 == 0)) {
6787                         /*
6788                          * Enter Path MTU Black-hole Detection mechanism: -
6789                          * Disable Path MTU Discovery (IP "DF" bit). -
6790                          * Reduce MTU to lower value than what we negotiated
6791                          * with peer.
6792                          */
6793                         if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) {
6794                                 /* Record that we may have found a black hole. */
6795                                 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE;
6796                                 /* Keep track of previous MSS. */
6797                                 tp->t_pmtud_saved_maxseg = tp->t_maxseg;
6798                         }
6799
6800                         /*
6801                          * Reduce the MSS to blackhole value or to the
6802                          * default in an attempt to retransmit.
6803                          */
6804 #ifdef INET6
6805                         if (isipv6 &&
6806                             tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {
6807                                 /* Use the sysctl tuneable blackhole MSS. */
6808                                 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;
6809                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
6810                         } else if (isipv6) {
6811                                 /* Use the default MSS. */
6812                                 tp->t_maxseg = V_tcp_v6mssdflt;
6813                                 /*
6814                                  * Disable Path MTU Discovery when we switch
6815                                  * to minmss.
6816                                  */
6817                                 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
6818                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
6819                         }
6820 #endif
6821 #if defined(INET6) && defined(INET)
6822                         else
6823 #endif
6824 #ifdef INET
6825                         if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {
6826                                 /* Use the sysctl tuneable blackhole MSS. */
6827                                 tp->t_maxseg = V_tcp_pmtud_blackhole_mss;
6828                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
6829                         } else {
6830                                 /* Use the default MSS. */
6831                                 tp->t_maxseg = V_tcp_mssdflt;
6832                                 /*
6833                                  * Disable Path MTU Discovery when we switch
6834                                  * to minmss.
6835                                  */
6836                                 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
6837                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
6838                         }
6839 #endif
6840                 } else {
6841                         /*
6842                          * If further retransmissions are still unsuccessful
6843                          * with a lowered MTU, maybe this isn't a blackhole
6844                          * and we restore the previous MSS and blackhole
6845                          * detection flags. The limit '6' is determined by
6846                          * giving each probe stage (1448, 1188, 524) 2
6847                          * chances to recover.
6848                          */
6849                         if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&
6850                             (tp->t_rxtshift >= 6)) {
6851                                 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
6852                                 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;
6853                                 tp->t_maxseg = tp->t_pmtud_saved_maxseg;
6854                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_failed);
6855                         }
6856                 }
6857         }
6858         /*
6859          * Disable RFC1323 and SACK if we haven't got any response to
6860          * our third SYN to work-around some broken terminal servers
6861          * (most of which have hopefully been retired) that have bad VJ
6862          * header compression code which trashes TCP segments containing
6863          * unknown-to-them TCP options.
6864          */
6865         if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&
6866             (tp->t_rxtshift == 3))
6867                 tp->t_flags &= ~(TF_REQ_SCALE|TF_REQ_TSTMP|TF_SACK_PERMIT);
6868         /*
6869          * If we backed off this far, our srtt estimate is probably bogus.
6870          * Clobber it so we'll take the next rtt measurement as our srtt;
6871          * move the current srtt into rttvar to keep the current retransmit
6872          * times until then.
6873          */
6874         if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
6875 #ifdef INET6
6876                 if ((tp->t_inpcb->inp_vflag & INP_IPV6) != 0)
6877                         in6_losing(tp->t_inpcb);
6878                 else
6879 #endif
6880                         in_losing(tp->t_inpcb);
6881                 tp->t_rttvar += tp->t_srtt;
6882                 tp->t_srtt = 0;
6883         }
6884         sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
6885         tp->snd_recover = tp->snd_max;
6886         tp->t_flags |= TF_ACKNOW;
6887         tp->t_rtttime = 0;
6888         rack_cong_signal(tp, CC_RTO, tp->snd_una, __LINE__);
6889 out:
6890         return (retval);
6891 }
6892
6893 static int
6894 rack_process_timers(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, uint8_t hpts_calling, uint8_t *doing_tlp)
6895 {
6896         int32_t ret = 0;
6897         int32_t timers = (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK);
6898
6899         if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
6900             (tp->t_flags & TF_GPUTINPROG)) {
6901                 /*
6902                  * We have a goodput in progress
6903                  * and we have entered a late state.
6904                  * Do we have enough data in the sb
6905                  * to handle the GPUT request?
6906                  */
6907                 uint32_t bytes;
6908
6909                 bytes = tp->gput_ack - tp->gput_seq;
6910                 if (SEQ_GT(tp->gput_seq, tp->snd_una))
6911                         bytes += tp->gput_seq - tp->snd_una;
6912                 if (bytes > sbavail(&tp->t_inpcb->inp_socket->so_snd)) {
6913                         /*
6914                          * There are not enough bytes in the socket
6915                          * buffer that have been sent to cover this
6916                          * measurement. Cancel it.
6917                          */
6918                         rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
6919                                                    rack->r_ctl.rc_gp_srtt /*flex1*/,
6920                                                    tp->gput_seq,
6921                                                    0, 0, 18, __LINE__, NULL, 0);
6922                         tp->t_flags &= ~TF_GPUTINPROG;
6923                 }
6924         }
6925         if (timers == 0) {
6926                 return (0);
6927         }
6928         if (tp->t_state == TCPS_LISTEN) {
6929                 /* no timers on listen sockets */
6930                 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)
6931                         return (0);
6932                 return (1);
6933         }
6934         if ((timers & PACE_TMR_RACK) &&
6935             rack->rc_on_min_to) {
6936                 /*
6937                  * For the rack timer when we
6938                  * are on a min-timeout (which means rrr_conf = 3)
6939                  * we don't want to check the timer. It may
6940                  * be going off for a pace and thats ok we
6941                  * want to send the retransmit (if its ready).
6942                  *
6943                  * If its on a normal rack timer (non-min) then
6944                  * we will check if its expired.
6945                  */
6946                 goto skip_time_check;
6947         }
6948         if (TSTMP_LT(cts, rack->r_ctl.rc_timer_exp)) {
6949                 uint32_t left;
6950
6951                 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
6952                         ret = -1;
6953                         rack_log_to_processing(rack, cts, ret, 0);
6954                         return (0);
6955                 }
6956                 if (hpts_calling == 0) {
6957                         /*
6958                          * A user send or queued mbuf (sack) has called us? We
6959                          * return 0 and let the pacing guards
6960                          * deal with it if they should or
6961                          * should not cause a send.
6962                          */
6963                         ret = -2;
6964                         rack_log_to_processing(rack, cts, ret, 0);
6965                         return (0);
6966                 }
6967                 /*
6968                  * Ok our timer went off early and we are not paced false
6969                  * alarm, go back to sleep.
6970                  */
6971                 ret = -3;
6972                 left = rack->r_ctl.rc_timer_exp - cts;
6973                 tcp_hpts_insert(tp->t_inpcb, HPTS_MS_TO_SLOTS(left));
6974                 rack_log_to_processing(rack, cts, ret, left);
6975                 return (1);
6976         }
6977 skip_time_check:
6978         rack->rc_tmr_stopped = 0;
6979         rack->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK;
6980         if (timers & PACE_TMR_DELACK) {
6981                 ret = rack_timeout_delack(tp, rack, cts);
6982         } else if (timers & PACE_TMR_RACK) {
6983                 rack->r_ctl.rc_tlp_rxt_last_time = cts;
6984                 rack->r_fast_output = 0;
6985                 ret = rack_timeout_rack(tp, rack, cts);
6986         } else if (timers & PACE_TMR_TLP) {
6987                 rack->r_ctl.rc_tlp_rxt_last_time = cts;
6988                 ret = rack_timeout_tlp(tp, rack, cts, doing_tlp);
6989         } else if (timers & PACE_TMR_RXT) {
6990                 rack->r_ctl.rc_tlp_rxt_last_time = cts;
6991                 rack->r_fast_output = 0;
6992                 ret = rack_timeout_rxt(tp, rack, cts);
6993         } else if (timers & PACE_TMR_PERSIT) {
6994                 ret = rack_timeout_persist(tp, rack, cts);
6995         } else if (timers & PACE_TMR_KEEP) {
6996                 ret = rack_timeout_keepalive(tp, rack, cts);
6997         }
6998         rack_log_to_processing(rack, cts, ret, timers);
6999         return (ret);
7000 }
7001
7002 static void
7003 rack_timer_cancel(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, int line)
7004 {
7005         struct timeval tv;
7006         uint32_t us_cts, flags_on_entry;
7007         uint8_t hpts_removed = 0;
7008
7009         flags_on_entry = rack->r_ctl.rc_hpts_flags;
7010         us_cts = tcp_get_usecs(&tv);
7011         if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
7012             ((TSTMP_GEQ(us_cts, rack->r_ctl.rc_last_output_to)) ||
7013              ((tp->snd_max - tp->snd_una) == 0))) {
7014                 tcp_hpts_remove(rack->rc_inp);
7015                 hpts_removed = 1;
7016                 /* If we were not delayed cancel out the flag. */
7017                 if ((tp->snd_max - tp->snd_una) == 0)
7018                         rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
7019                 rack_log_to_cancel(rack, hpts_removed, line, us_cts, &tv, flags_on_entry);
7020         }
7021         if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
7022                 rack->rc_tmr_stopped = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
7023                 if (tcp_in_hpts(rack->rc_inp) &&
7024                     ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)) {
7025                         /*
7026                          * Canceling timer's when we have no output being
7027                          * paced. We also must remove ourselves from the
7028                          * hpts.
7029                          */
7030                         tcp_hpts_remove(rack->rc_inp);
7031                         hpts_removed = 1;
7032                 }
7033                 rack->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);
7034         }
7035         if (hpts_removed == 0)
7036                 rack_log_to_cancel(rack, hpts_removed, line, us_cts, &tv, flags_on_entry);
7037 }
7038
7039 static void
7040 rack_timer_stop(struct tcpcb *tp, uint32_t timer_type)
7041 {
7042         return;
7043 }
7044
7045 static int
7046 rack_stopall(struct tcpcb *tp)
7047 {
7048         struct tcp_rack *rack;
7049         rack = (struct tcp_rack *)tp->t_fb_ptr;
7050         rack->t_timers_stopped = 1;
7051         return (0);
7052 }
7053
7054 static void
7055 rack_timer_activate(struct tcpcb *tp, uint32_t timer_type, uint32_t delta)
7056 {
7057         return;
7058 }
7059
7060 static int
7061 rack_timer_active(struct tcpcb *tp, uint32_t timer_type)
7062 {
7063         return (0);
7064 }
7065
7066 static void
7067 rack_stop_all_timers(struct tcpcb *tp)
7068 {
7069         struct tcp_rack *rack;
7070
7071         /*
7072          * Assure no timers are running.
7073          */
7074         if (tcp_timer_active(tp, TT_PERSIST)) {
7075                 /* We enter in persists, set the flag appropriately */
7076                 rack = (struct tcp_rack *)tp->t_fb_ptr;
7077                 rack->rc_in_persist = 1;
7078         }
7079         tcp_timer_suspend(tp, TT_PERSIST);
7080         tcp_timer_suspend(tp, TT_REXMT);
7081         tcp_timer_suspend(tp, TT_KEEP);
7082         tcp_timer_suspend(tp, TT_DELACK);
7083 }
7084
7085 static void
7086 rack_update_rsm(struct tcpcb *tp, struct tcp_rack *rack,
7087     struct rack_sendmap *rsm, uint64_t ts, uint16_t add_flag)
7088 {
7089         int32_t idx;
7090
7091         rsm->r_rtr_cnt++;
7092         rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
7093         rsm->r_dupack = 0;
7094         if (rsm->r_rtr_cnt > RACK_NUM_OF_RETRANS) {
7095                 rsm->r_rtr_cnt = RACK_NUM_OF_RETRANS;
7096                 rsm->r_flags |= RACK_OVERMAX;
7097         }
7098         if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & RACK_TLP) == 0)) {
7099                 rack->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
7100                 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);
7101         }
7102         idx = rsm->r_rtr_cnt - 1;
7103         rsm->r_tim_lastsent[idx] = ts;
7104         /*
7105          * Here we don't add in the len of send, since its already
7106          * in snduna <->snd_max.
7107          */
7108         rsm->r_fas = ctf_flight_size(rack->rc_tp,
7109                                      rack->r_ctl.rc_sacked);
7110         if (rsm->r_flags & RACK_ACKED) {
7111                 /* Problably MTU discovery messing with us */
7112                 rsm->r_flags &= ~RACK_ACKED;
7113                 rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7114         }
7115         if (rsm->r_in_tmap) {
7116                 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
7117                 rsm->r_in_tmap = 0;
7118         }
7119         TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext);
7120         rsm->r_in_tmap = 1;
7121         /* Take off the must retransmit flag, if its on */
7122         if (rsm->r_flags & RACK_MUST_RXT) {
7123                 if (rack->r_must_retran)
7124                         rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start);
7125                 if (SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) {
7126                         /*
7127                          * We have retransmitted all we need. Clear
7128                          * any must retransmit flags.
7129                          */
7130                         rack->r_must_retran = 0;
7131                         rack->r_ctl.rc_out_at_rto = 0;
7132                 }
7133                 rsm->r_flags &= ~RACK_MUST_RXT;
7134         }
7135         if (rsm->r_flags & RACK_SACK_PASSED) {
7136                 /* We have retransmitted due to the SACK pass */
7137                 rsm->r_flags &= ~RACK_SACK_PASSED;
7138                 rsm->r_flags |= RACK_WAS_SACKPASS;
7139         }
7140 }
7141
7142 static uint32_t
7143 rack_update_entry(struct tcpcb *tp, struct tcp_rack *rack,
7144     struct rack_sendmap *rsm, uint64_t ts, int32_t *lenp, uint16_t add_flag)
7145 {
7146         /*
7147          * We (re-)transmitted starting at rsm->r_start for some length
7148          * (possibly less than r_end.
7149          */
7150         struct rack_sendmap *nrsm;
7151 #ifdef INVARIANTS
7152         struct rack_sendmap *insret;
7153 #endif
7154         uint32_t c_end;
7155         int32_t len;
7156
7157         len = *lenp;
7158         c_end = rsm->r_start + len;
7159         if (SEQ_GEQ(c_end, rsm->r_end)) {
7160                 /*
7161                  * We retransmitted the whole piece or more than the whole
7162                  * slopping into the next rsm.
7163                  */
7164                 rack_update_rsm(tp, rack, rsm, ts, add_flag);
7165                 if (c_end == rsm->r_end) {
7166                         *lenp = 0;
7167                         return (0);
7168                 } else {
7169                         int32_t act_len;
7170
7171                         /* Hangs over the end return whats left */
7172                         act_len = rsm->r_end - rsm->r_start;
7173                         *lenp = (len - act_len);
7174                         return (rsm->r_end);
7175                 }
7176                 /* We don't get out of this block. */
7177         }
7178         /*
7179          * Here we retransmitted less than the whole thing which means we
7180          * have to split this into what was transmitted and what was not.
7181          */
7182         nrsm = rack_alloc_full_limit(rack);
7183         if (nrsm == NULL) {
7184                 /*
7185                  * We can't get memory, so lets not proceed.
7186                  */
7187                 *lenp = 0;
7188                 return (0);
7189         }
7190         /*
7191          * So here we are going to take the original rsm and make it what we
7192          * retransmitted. nrsm will be the tail portion we did not
7193          * retransmit. For example say the chunk was 1, 11 (10 bytes). And
7194          * we retransmitted 5 bytes i.e. 1, 5. The original piece shrinks to
7195          * 1, 6 and the new piece will be 6, 11.
7196          */
7197         rack_clone_rsm(rack, nrsm, rsm, c_end);
7198         nrsm->r_dupack = 0;
7199         rack_log_retran_reason(rack, nrsm, __LINE__, 0, 2);
7200 #ifndef INVARIANTS
7201         (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
7202 #else
7203         insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
7204         if (insret != NULL) {
7205                 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p",
7206                       nrsm, insret, rack, rsm);
7207         }
7208 #endif
7209         if (rsm->r_in_tmap) {
7210                 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7211                 nrsm->r_in_tmap = 1;
7212         }
7213         rsm->r_flags &= (~RACK_HAS_FIN);
7214         rack_update_rsm(tp, rack, rsm, ts, add_flag);
7215         /* Log a split of rsm into rsm and nrsm */
7216         rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 0, __LINE__);
7217         *lenp = 0;
7218         return (0);
7219 }
7220
7221 static void
7222 rack_log_output(struct tcpcb *tp, struct tcpopt *to, int32_t len,
7223                 uint32_t seq_out, uint16_t th_flags, int32_t err, uint64_t cts,
7224                 struct rack_sendmap *hintrsm, uint16_t add_flag, struct mbuf *s_mb, uint32_t s_moff, int hw_tls)
7225 {
7226         struct tcp_rack *rack;
7227         struct rack_sendmap *rsm, *nrsm, fe;
7228 #ifdef INVARIANTS
7229         struct rack_sendmap *insret;
7230 #endif
7231         register uint32_t snd_max, snd_una;
7232
7233         /*
7234          * Add to the RACK log of packets in flight or retransmitted. If
7235          * there is a TS option we will use the TS echoed, if not we will
7236          * grab a TS.
7237          *
7238          * Retransmissions will increment the count and move the ts to its
7239          * proper place. Note that if options do not include TS's then we
7240          * won't be able to effectively use the ACK for an RTT on a retran.
7241          *
7242          * Notes about r_start and r_end. Lets consider a send starting at
7243          * sequence 1 for 10 bytes. In such an example the r_start would be
7244          * 1 (starting sequence) but the r_end would be r_start+len i.e. 11.
7245          * This means that r_end is actually the first sequence for the next
7246          * slot (11).
7247          *
7248          */
7249         /*
7250          * If err is set what do we do XXXrrs? should we not add the thing?
7251          * -- i.e. return if err != 0 or should we pretend we sent it? --
7252          * i.e. proceed with add ** do this for now.
7253          */
7254         INP_WLOCK_ASSERT(tp->t_inpcb);
7255         if (err)
7256                 /*
7257                  * We don't log errors -- we could but snd_max does not
7258                  * advance in this case either.
7259                  */
7260                 return;
7261
7262         if (th_flags & TH_RST) {
7263                 /*
7264                  * We don't log resets and we return immediately from
7265                  * sending
7266                  */
7267                 return;
7268         }
7269         rack = (struct tcp_rack *)tp->t_fb_ptr;
7270         snd_una = tp->snd_una;
7271         snd_max = tp->snd_max;
7272         if (th_flags & (TH_SYN | TH_FIN)) {
7273                 /*
7274                  * The call to rack_log_output is made before bumping
7275                  * snd_max. This means we can record one extra byte on a SYN
7276                  * or FIN if seq_out is adding more on and a FIN is present
7277                  * (and we are not resending).
7278                  */
7279                 if ((th_flags & TH_SYN) && (seq_out == tp->iss))
7280                         len++;
7281                 if (th_flags & TH_FIN)
7282                         len++;
7283                 if (SEQ_LT(snd_max, tp->snd_nxt)) {
7284                         /*
7285                          * The add/update as not been done for the FIN/SYN
7286                          * yet.
7287                          */
7288                         snd_max = tp->snd_nxt;
7289                 }
7290         }
7291         if (SEQ_LEQ((seq_out + len), snd_una)) {
7292                 /* Are sending an old segment to induce an ack (keep-alive)? */
7293                 return;
7294         }
7295         if (SEQ_LT(seq_out, snd_una)) {
7296                 /* huh? should we panic? */
7297                 uint32_t end;
7298
7299                 end = seq_out + len;
7300                 seq_out = snd_una;
7301                 if (SEQ_GEQ(end, seq_out))
7302                         len = end - seq_out;
7303                 else
7304                         len = 0;
7305         }
7306         if (len == 0) {
7307                 /* We don't log zero window probes */
7308                 return;
7309         }
7310         if (IN_FASTRECOVERY(tp->t_flags)) {
7311                 rack->r_ctl.rc_prr_out += len;
7312         }
7313         /* First question is it a retransmission or new? */
7314         if (seq_out == snd_max) {
7315                 /* Its new */
7316 again:
7317                 rsm = rack_alloc(rack);
7318                 if (rsm == NULL) {
7319                         /*
7320                          * Hmm out of memory and the tcb got destroyed while
7321                          * we tried to wait.
7322                          */
7323                         return;
7324                 }
7325                 if (th_flags & TH_FIN) {
7326                         rsm->r_flags = RACK_HAS_FIN|add_flag;
7327                 } else {
7328                         rsm->r_flags = add_flag;
7329                 }
7330                 if (hw_tls)
7331                         rsm->r_hw_tls = 1;
7332                 rsm->r_tim_lastsent[0] = cts;
7333                 rsm->r_rtr_cnt = 1;
7334                 rsm->r_rtr_bytes = 0;
7335                 if (th_flags & TH_SYN) {
7336                         /* The data space is one beyond snd_una */
7337                         rsm->r_flags |= RACK_HAS_SYN;
7338                 }
7339                 rsm->r_start = seq_out;
7340                 rsm->r_end = rsm->r_start + len;
7341                 rsm->r_dupack = 0;
7342                 /*
7343                  * save off the mbuf location that
7344                  * sndmbuf_noadv returned (which is
7345                  * where we started copying from)..
7346                  */
7347                 rsm->m = s_mb;
7348                 rsm->soff = s_moff;
7349                 /*
7350                  * Here we do add in the len of send, since its not yet
7351                  * reflected in in snduna <->snd_max
7352                  */
7353                 rsm->r_fas = (ctf_flight_size(rack->rc_tp,
7354                                               rack->r_ctl.rc_sacked) +
7355                               (rsm->r_end - rsm->r_start));
7356                 /* rsm->m will be NULL if RACK_HAS_SYN or RACK_HAS_FIN is set */
7357                 if (rsm->m) {
7358                         if (rsm->m->m_len <= rsm->soff) {
7359                                 /*
7360                                  * XXXrrs Question, will this happen?
7361                                  *
7362                                  * If sbsndptr is set at the correct place
7363                                  * then s_moff should always be somewhere
7364                                  * within rsm->m. But if the sbsndptr was
7365                                  * off then that won't be true. If it occurs
7366                                  * we need to walkout to the correct location.
7367                                  */
7368                                 struct mbuf *lm;
7369
7370                                 lm = rsm->m;
7371                                 while (lm->m_len <= rsm->soff) {
7372                                         rsm->soff -= lm->m_len;
7373                                         lm = lm->m_next;
7374                                         KASSERT(lm != NULL, ("%s rack:%p lm goes null orig_off:%u origmb:%p rsm->soff:%u",
7375                                                              __func__, rack, s_moff, s_mb, rsm->soff));
7376                                 }
7377                                 rsm->m = lm;
7378                         }
7379                         rsm->orig_m_len = rsm->m->m_len;
7380                 } else
7381                         rsm->orig_m_len = 0;
7382                 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
7383                 /* Log a new rsm */
7384                 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_NEW, 0, __LINE__);
7385 #ifndef INVARIANTS
7386                 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
7387 #else
7388                 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
7389                 if (insret != NULL) {
7390                         panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p",
7391                               nrsm, insret, rack, rsm);
7392                 }
7393 #endif
7394                 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext);
7395                 rsm->r_in_tmap = 1;
7396                 /*
7397                  * Special case detection, is there just a single
7398                  * packet outstanding when we are not in recovery?
7399                  *
7400                  * If this is true mark it so.
7401                  */
7402                 if ((IN_FASTRECOVERY(tp->t_flags) == 0) &&
7403                     (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) == ctf_fixed_maxseg(tp))) {
7404                         struct rack_sendmap *prsm;
7405
7406                         prsm = RB_PREV(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
7407                         if (prsm)
7408                                 prsm->r_one_out_nr = 1;
7409                 }
7410                 return;
7411         }
7412         /*
7413          * If we reach here its a retransmission and we need to find it.
7414          */
7415         memset(&fe, 0, sizeof(fe));
7416 more:
7417         if (hintrsm && (hintrsm->r_start == seq_out)) {
7418                 rsm = hintrsm;
7419                 hintrsm = NULL;
7420         } else {
7421                 /* No hints sorry */
7422                 rsm = NULL;
7423         }
7424         if ((rsm) && (rsm->r_start == seq_out)) {
7425                 seq_out = rack_update_entry(tp, rack, rsm, cts, &len, add_flag);
7426                 if (len == 0) {
7427                         return;
7428                 } else {
7429                         goto more;
7430                 }
7431         }
7432         /* Ok it was not the last pointer go through it the hard way. */
7433 refind:
7434         fe.r_start = seq_out;
7435         rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe);
7436         if (rsm) {
7437                 if (rsm->r_start == seq_out) {
7438                         seq_out = rack_update_entry(tp, rack, rsm, cts, &len, add_flag);
7439                         if (len == 0) {
7440                                 return;
7441                         } else {
7442                                 goto refind;
7443                         }
7444                 }
7445                 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) {
7446                         /* Transmitted within this piece */
7447                         /*
7448                          * Ok we must split off the front and then let the
7449                          * update do the rest
7450                          */
7451                         nrsm = rack_alloc_full_limit(rack);
7452                         if (nrsm == NULL) {
7453                                 rack_update_rsm(tp, rack, rsm, cts, add_flag);
7454                                 return;
7455                         }
7456                         /*
7457                          * copy rsm to nrsm and then trim the front of rsm
7458                          * to not include this part.
7459                          */
7460                         rack_clone_rsm(rack, nrsm, rsm, seq_out);
7461                         rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SPLIT, 0, __LINE__);
7462 #ifndef INVARIANTS
7463                         (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
7464 #else
7465                         insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
7466                         if (insret != NULL) {
7467                                 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p",
7468                                       nrsm, insret, rack, rsm);
7469                         }
7470 #endif
7471                         if (rsm->r_in_tmap) {
7472                                 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7473                                 nrsm->r_in_tmap = 1;
7474                         }
7475                         rsm->r_flags &= (~RACK_HAS_FIN);
7476                         seq_out = rack_update_entry(tp, rack, nrsm, cts, &len, add_flag);
7477                         if (len == 0) {
7478                                 return;
7479                         } else if (len > 0)
7480                                 goto refind;
7481                 }
7482         }
7483         /*
7484          * Hmm not found in map did they retransmit both old and on into the
7485          * new?
7486          */
7487         if (seq_out == tp->snd_max) {
7488                 goto again;
7489         } else if (SEQ_LT(seq_out, tp->snd_max)) {
7490 #ifdef INVARIANTS
7491                 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n",
7492                        seq_out, len, tp->snd_una, tp->snd_max);
7493                 printf("Starting Dump of all rack entries\n");
7494                 RB_FOREACH(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) {
7495                         printf("rsm:%p start:%u end:%u\n",
7496                                rsm, rsm->r_start, rsm->r_end);
7497                 }
7498                 printf("Dump complete\n");
7499                 panic("seq_out not found rack:%p tp:%p",
7500                       rack, tp);
7501 #endif
7502         } else {
7503 #ifdef INVARIANTS
7504                 /*
7505                  * Hmm beyond sndmax? (only if we are using the new rtt-pack
7506                  * flag)
7507                  */
7508                 panic("seq_out:%u(%d) is beyond snd_max:%u tp:%p",
7509                       seq_out, len, tp->snd_max, tp);
7510 #endif
7511         }
7512 }
7513
7514 /*
7515  * Record one of the RTT updates from an ack into
7516  * our sample structure.
7517  */
7518
7519 static void
7520 tcp_rack_xmit_timer(struct tcp_rack *rack, int32_t rtt, uint32_t len, uint32_t us_rtt,
7521                     int confidence, struct rack_sendmap *rsm, uint16_t rtrcnt)
7522 {
7523         if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) ||
7524             (rack->r_ctl.rack_rs.rs_rtt_lowest > rtt)) {
7525                 rack->r_ctl.rack_rs.rs_rtt_lowest = rtt;
7526         }
7527         if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) ||
7528             (rack->r_ctl.rack_rs.rs_rtt_highest < rtt)) {
7529                 rack->r_ctl.rack_rs.rs_rtt_highest = rtt;
7530         }
7531         if (rack->rc_tp->t_flags & TF_GPUTINPROG) {
7532             if (us_rtt < rack->r_ctl.rc_gp_lowrtt)
7533                 rack->r_ctl.rc_gp_lowrtt = us_rtt;
7534             if (rack->rc_tp->snd_wnd > rack->r_ctl.rc_gp_high_rwnd)
7535                     rack->r_ctl.rc_gp_high_rwnd = rack->rc_tp->snd_wnd;
7536         }
7537         if ((confidence == 1) &&
7538             ((rsm == NULL) ||
7539              (rsm->r_just_ret) ||
7540              (rsm->r_one_out_nr &&
7541               len < (ctf_fixed_maxseg(rack->rc_tp) * 2)))) {
7542                 /*
7543                  * If the rsm had a just return
7544                  * hit it then we can't trust the
7545                  * rtt measurement for buffer deterimination
7546                  * Note that a confidence of 2, indicates
7547                  * SACK'd which overrides the r_just_ret or
7548                  * the r_one_out_nr. If it was a CUM-ACK and
7549                  * we had only two outstanding, but get an
7550                  * ack for only 1. Then that also lowers our
7551                  * confidence.
7552                  */
7553                 confidence = 0;
7554         }
7555         if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY) ||
7556             (rack->r_ctl.rack_rs.rs_us_rtt > us_rtt)) {
7557                 if (rack->r_ctl.rack_rs.confidence == 0) {
7558                         /*
7559                          * We take anything with no current confidence
7560                          * saved.
7561                          */
7562                         rack->r_ctl.rack_rs.rs_us_rtt = us_rtt;
7563                         rack->r_ctl.rack_rs.confidence = confidence;
7564                         rack->r_ctl.rack_rs.rs_us_rtrcnt = rtrcnt;
7565                 } else if (confidence || rack->r_ctl.rack_rs.confidence) {
7566                         /*
7567                          * Once we have a confident number,
7568                          * we can update it with a smaller
7569                          * value since this confident number
7570                          * may include the DSACK time until
7571                          * the next segment (the second one) arrived.
7572                          */
7573                         rack->r_ctl.rack_rs.rs_us_rtt = us_rtt;
7574                         rack->r_ctl.rack_rs.confidence = confidence;
7575                         rack->r_ctl.rack_rs.rs_us_rtrcnt = rtrcnt;
7576                 }
7577         }
7578         rack_log_rtt_upd(rack->rc_tp, rack, us_rtt, len, rsm, confidence);
7579         rack->r_ctl.rack_rs.rs_flags = RACK_RTT_VALID;
7580         rack->r_ctl.rack_rs.rs_rtt_tot += rtt;
7581         rack->r_ctl.rack_rs.rs_rtt_cnt++;
7582 }
7583
7584 /*
7585  * Collect new round-trip time estimate
7586  * and update averages and current timeout.
7587  */
7588 static void
7589 tcp_rack_xmit_timer_commit(struct tcp_rack *rack, struct tcpcb *tp)
7590 {
7591         int32_t delta;
7592         int32_t rtt;
7593
7594         if (rack->r_ctl.rack_rs.rs_flags & RACK_RTT_EMPTY)
7595                 /* No valid sample */
7596                 return;
7597         if (rack->r_ctl.rc_rate_sample_method == USE_RTT_LOW) {
7598                 /* We are to use the lowest RTT seen in a single ack */
7599                 rtt = rack->r_ctl.rack_rs.rs_rtt_lowest;
7600         } else if (rack->r_ctl.rc_rate_sample_method == USE_RTT_HIGH) {
7601                 /* We are to use the highest RTT seen in a single ack */
7602                 rtt = rack->r_ctl.rack_rs.rs_rtt_highest;
7603         } else if (rack->r_ctl.rc_rate_sample_method == USE_RTT_AVG) {
7604                 /* We are to use the average RTT seen in a single ack */
7605                 rtt = (int32_t)(rack->r_ctl.rack_rs.rs_rtt_tot /
7606                                 (uint64_t)rack->r_ctl.rack_rs.rs_rtt_cnt);
7607         } else {
7608 #ifdef INVARIANTS
7609                 panic("Unknown rtt variant %d", rack->r_ctl.rc_rate_sample_method);
7610 #endif
7611                 return;
7612         }
7613         if (rtt == 0)
7614                 rtt = 1;
7615         if (rack->rc_gp_rtt_set == 0) {
7616                 /*
7617                  * With no RTT we have to accept
7618                  * even one we are not confident of.
7619                  */
7620                 rack->r_ctl.rc_gp_srtt = rack->r_ctl.rack_rs.rs_us_rtt;
7621                 rack->rc_gp_rtt_set = 1;
7622         } else if (rack->r_ctl.rack_rs.confidence) {
7623                 /* update the running gp srtt */
7624                 rack->r_ctl.rc_gp_srtt -= (rack->r_ctl.rc_gp_srtt/8);
7625                 rack->r_ctl.rc_gp_srtt += rack->r_ctl.rack_rs.rs_us_rtt / 8;
7626         }
7627         if (rack->r_ctl.rack_rs.confidence) {
7628                 /*
7629                  * record the low and high for highly buffered path computation,
7630                  * we only do this if we are confident (not a retransmission).
7631                  */
7632                 if (rack->r_ctl.rc_highest_us_rtt < rack->r_ctl.rack_rs.rs_us_rtt) {
7633                         rack->r_ctl.rc_highest_us_rtt = rack->r_ctl.rack_rs.rs_us_rtt;
7634                 }
7635                 if (rack->rc_highly_buffered == 0) {
7636                         /*
7637                          * Currently once we declare a path has
7638                          * highly buffered there is no going
7639                          * back, which may be a problem...
7640                          */
7641                         if ((rack->r_ctl.rc_highest_us_rtt / rack->r_ctl.rc_lowest_us_rtt) > rack_hbp_thresh) {
7642                                 rack_log_rtt_shrinks(rack, rack->r_ctl.rack_rs.rs_us_rtt,
7643                                                      rack->r_ctl.rc_highest_us_rtt,
7644                                                      rack->r_ctl.rc_lowest_us_rtt,
7645                                                      RACK_RTTS_SEEHBP);
7646                                 rack->rc_highly_buffered = 1;
7647                         }
7648                 }
7649         }
7650         if ((rack->r_ctl.rack_rs.confidence) ||
7651             (rack->r_ctl.rack_rs.rs_us_rtrcnt == 1)) {
7652                 /*
7653                  * If we are highly confident of it <or> it was
7654                  * never retransmitted we accept it as the last us_rtt.
7655                  */
7656                 rack->r_ctl.rc_last_us_rtt = rack->r_ctl.rack_rs.rs_us_rtt;
7657                 /* The lowest rtt can be set if its was not retransmited */
7658                 if (rack->r_ctl.rc_lowest_us_rtt > rack->r_ctl.rack_rs.rs_us_rtt) {
7659                         rack->r_ctl.rc_lowest_us_rtt = rack->r_ctl.rack_rs.rs_us_rtt;
7660                         if (rack->r_ctl.rc_lowest_us_rtt == 0)
7661                                 rack->r_ctl.rc_lowest_us_rtt = 1;
7662                 }
7663         }
7664         rack = (struct tcp_rack *)tp->t_fb_ptr;
7665         if (tp->t_srtt != 0) {
7666                 /*
7667                  * We keep a simple srtt in microseconds, like our rtt
7668                  * measurement. We don't need to do any tricks with shifting
7669                  * etc. Instead we just add in 1/8th of the new measurement
7670                  * and subtract out 1/8 of the old srtt. We do the same with
7671                  * the variance after finding the absolute value of the
7672                  * difference between this sample and the current srtt.
7673                  */
7674                 delta = tp->t_srtt - rtt;
7675                 /* Take off 1/8th of the current sRTT */
7676                 tp->t_srtt -= (tp->t_srtt >> 3);
7677                 /* Add in 1/8th of the new RTT just measured */
7678                 tp->t_srtt += (rtt >> 3);
7679                 if (tp->t_srtt <= 0)
7680                         tp->t_srtt = 1;
7681                 /* Now lets make the absolute value of the variance */
7682                 if (delta < 0)
7683                         delta = -delta;
7684                 /* Subtract out 1/8th */
7685                 tp->t_rttvar -= (tp->t_rttvar >> 3);
7686                 /* Add in 1/8th of the new variance we just saw */
7687                 tp->t_rttvar += (delta >> 3);
7688                 if (tp->t_rttvar <= 0)
7689                         tp->t_rttvar = 1;
7690                 if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar)
7691                         tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
7692         } else {
7693                 /*
7694                  * No rtt measurement yet - use the unsmoothed rtt. Set the
7695                  * variance to half the rtt (so our first retransmit happens
7696                  * at 3*rtt).
7697                  */
7698                 tp->t_srtt = rtt;
7699                 tp->t_rttvar = rtt >> 1;
7700                 tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
7701         }
7702         rack->rc_srtt_measure_made = 1;
7703         KMOD_TCPSTAT_INC(tcps_rttupdated);
7704         tp->t_rttupdated++;
7705 #ifdef STATS
7706         if (rack_stats_gets_ms_rtt == 0) {
7707                 /* Send in the microsecond rtt used for rxt timeout purposes */
7708                 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt));
7709         } else if (rack_stats_gets_ms_rtt == 1) {
7710                 /* Send in the millisecond rtt used for rxt timeout purposes */
7711                 int32_t ms_rtt;
7712
7713                 /* Round up */
7714                 ms_rtt = (rtt + HPTS_USEC_IN_MSEC - 1) / HPTS_USEC_IN_MSEC;
7715                 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, ms_rtt));
7716         } else if (rack_stats_gets_ms_rtt == 2) {
7717                 /* Send in the millisecond rtt has close to the path RTT as we can get  */
7718                 int32_t ms_rtt;
7719
7720                 /* Round up */
7721                 ms_rtt = (rack->r_ctl.rack_rs.rs_us_rtt + HPTS_USEC_IN_MSEC - 1) / HPTS_USEC_IN_MSEC;
7722                 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, ms_rtt));
7723         }  else {
7724                 /* Send in the microsecond rtt has close to the path RTT as we can get  */
7725                 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rack->r_ctl.rack_rs.rs_us_rtt));
7726         }
7727
7728 #endif
7729         /*
7730          * the retransmit should happen at rtt + 4 * rttvar. Because of the
7731          * way we do the smoothing, srtt and rttvar will each average +1/2
7732          * tick of bias.  When we compute the retransmit timer, we want 1/2
7733          * tick of rounding and 1 extra tick because of +-1/2 tick
7734          * uncertainty in the firing of the timer.  The bias will give us
7735          * exactly the 1.5 tick we need.  But, because the bias is
7736          * statistical, we have to test that we don't drop below the minimum
7737          * feasible timer (which is 2 ticks).
7738          */
7739         tp->t_rxtshift = 0;
7740         RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
7741                       max(rack_rto_min, rtt + 2), rack_rto_max, rack->r_ctl.timer_slop);
7742         rack_log_rtt_sample(rack, rtt);
7743         tp->t_softerror = 0;
7744 }
7745
7746
7747 static void
7748 rack_apply_updated_usrtt(struct tcp_rack *rack, uint32_t us_rtt, uint32_t us_cts)
7749 {
7750         /*
7751          * Apply to filter the inbound us-rtt at us_cts.
7752          */
7753         uint32_t old_rtt;
7754
7755         old_rtt = get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt);
7756         apply_filter_min_small(&rack->r_ctl.rc_gp_min_rtt,
7757                                us_rtt, us_cts);
7758         if (old_rtt > us_rtt) {
7759                 /* We just hit a new lower rtt time */
7760                 rack_log_rtt_shrinks(rack,  us_cts,  old_rtt,
7761                                      __LINE__, RACK_RTTS_NEWRTT);
7762                 /*
7763                  * Only count it if its lower than what we saw within our
7764                  * calculated range.
7765                  */
7766                 if ((old_rtt - us_rtt) > rack_min_rtt_movement) {
7767                         if (rack_probertt_lower_within &&
7768                             rack->rc_gp_dyn_mul &&
7769                             (rack->use_fixed_rate == 0) &&
7770                             (rack->rc_always_pace)) {
7771                                 /*
7772                                  * We are seeing a new lower rtt very close
7773                                  * to the time that we would have entered probe-rtt.
7774                                  * This is probably due to the fact that a peer flow
7775                                  * has entered probe-rtt. Lets go in now too.
7776                                  */
7777                                 uint32_t val;
7778
7779                                 val = rack_probertt_lower_within * rack_time_between_probertt;
7780                                 val /= 100;
7781                                 if ((rack->in_probe_rtt == 0)  &&
7782                                     ((us_cts - rack->r_ctl.rc_lower_rtt_us_cts) >= (rack_time_between_probertt - val))) {
7783                                         rack_enter_probertt(rack, us_cts);
7784                                 }
7785                         }
7786                         rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
7787                 }
7788         }
7789 }
7790
7791 static int
7792 rack_update_rtt(struct tcpcb *tp, struct tcp_rack *rack,
7793     struct rack_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, tcp_seq th_ack)
7794 {
7795         uint32_t us_rtt;
7796         int32_t i, all;
7797         uint32_t t, len_acked;
7798
7799         if ((rsm->r_flags & RACK_ACKED) ||
7800             (rsm->r_flags & RACK_WAS_ACKED))
7801                 /* Already done */
7802                 return (0);
7803         if (rsm->r_no_rtt_allowed) {
7804                 /* Not allowed */
7805                 return (0);
7806         }
7807         if (ack_type == CUM_ACKED) {
7808                 if (SEQ_GT(th_ack, rsm->r_end)) {
7809                         len_acked = rsm->r_end - rsm->r_start;
7810                         all = 1;
7811                 } else {
7812                         len_acked = th_ack - rsm->r_start;
7813                         all = 0;
7814                 }
7815         } else {
7816                 len_acked = rsm->r_end - rsm->r_start;
7817                 all = 0;
7818         }
7819         if (rsm->r_rtr_cnt == 1) {
7820
7821                 t = cts - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
7822                 if ((int)t <= 0)
7823                         t = 1;
7824                 if (!tp->t_rttlow || tp->t_rttlow > t)
7825                         tp->t_rttlow = t;
7826                 if (!rack->r_ctl.rc_rack_min_rtt ||
7827                     SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) {
7828                         rack->r_ctl.rc_rack_min_rtt = t;
7829                         if (rack->r_ctl.rc_rack_min_rtt == 0) {
7830                                 rack->r_ctl.rc_rack_min_rtt = 1;
7831                         }
7832                 }
7833                 if (TSTMP_GT(tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time), rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]))
7834                         us_rtt = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time) - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
7835                 else
7836                         us_rtt = tcp_get_usecs(NULL) - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
7837                 if (us_rtt == 0)
7838                         us_rtt = 1;
7839                 if (CC_ALGO(tp)->rttsample != NULL) {
7840                         /* Kick the RTT to the CC */
7841                         CC_ALGO(tp)->rttsample(tp->ccv, us_rtt, 1, rsm->r_fas);
7842                 }
7843                 rack_apply_updated_usrtt(rack, us_rtt, tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time));
7844                 if (ack_type == SACKED) {
7845                         rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)], cts, 1);
7846                         tcp_rack_xmit_timer(rack, t + 1, len_acked, us_rtt, 2 , rsm, rsm->r_rtr_cnt);
7847                 } else {
7848                         /*
7849                          * We need to setup what our confidence
7850                          * is in this ack.
7851                          *
7852                          * If the rsm was app limited and it is
7853                          * less than a mss in length (the end
7854                          * of the send) then we have a gap. If we
7855                          * were app limited but say we were sending
7856                          * multiple MSS's then we are more confident
7857                          * int it.
7858                          *
7859                          * When we are not app-limited then we see if
7860                          * the rsm is being included in the current
7861                          * measurement, we tell this by the app_limited_needs_set
7862                          * flag.
7863                          *
7864                          * Note that being cwnd blocked is not applimited
7865                          * as well as the pacing delay between packets which
7866                          * are sending only 1 or 2 MSS's also will show up
7867                          * in the RTT. We probably need to examine this algorithm
7868                          * a bit more and enhance it to account for the delay
7869                          * between rsm's. We could do that by saving off the
7870                          * pacing delay of each rsm (in an rsm) and then
7871                          * factoring that in somehow though for now I am
7872                          * not sure how :)
7873                          */
7874                         int calc_conf = 0;
7875
7876                         if (rsm->r_flags & RACK_APP_LIMITED) {
7877                                 if (all && (len_acked <= ctf_fixed_maxseg(tp)))
7878                                         calc_conf = 0;
7879                                 else
7880                                         calc_conf = 1;
7881                         } else if (rack->app_limited_needs_set == 0) {
7882                                 calc_conf = 1;
7883                         } else {
7884                                 calc_conf = 0;
7885                         }
7886                         rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)], cts, 2);
7887                         tcp_rack_xmit_timer(rack, t + 1, len_acked, us_rtt,
7888                                             calc_conf, rsm, rsm->r_rtr_cnt);
7889                 }
7890                 if ((rsm->r_flags & RACK_TLP) &&
7891                     (!IN_FASTRECOVERY(tp->t_flags))) {
7892                         /* Segment was a TLP and our retrans matched */
7893                         if (rack->r_ctl.rc_tlp_cwnd_reduce) {
7894                                 rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__);
7895                         }
7896                 }
7897                 if (SEQ_LT(rack->r_ctl.rc_rack_tmit_time, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)])) {
7898                         /* New more recent rack_tmit_time */
7899                         rack->r_ctl.rc_rack_tmit_time = (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
7900                         rack->rc_rack_rtt = t;
7901                 }
7902                 return (1);
7903         }
7904         /*
7905          * We clear the soft/rxtshift since we got an ack.
7906          * There is no assurance we will call the commit() function
7907          * so we need to clear these to avoid incorrect handling.
7908          */
7909         tp->t_rxtshift = 0;
7910         RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
7911                       rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
7912         tp->t_softerror = 0;
7913         if (to && (to->to_flags & TOF_TS) &&
7914             (ack_type == CUM_ACKED) &&
7915             (to->to_tsecr) &&
7916             ((rsm->r_flags & RACK_OVERMAX) == 0)) {
7917                 /*
7918                  * Now which timestamp does it match? In this block the ACK
7919                  * must be coming from a previous transmission.
7920                  */
7921                 for (i = 0; i < rsm->r_rtr_cnt; i++) {
7922                         if (rack_ts_to_msec(rsm->r_tim_lastsent[i]) == to->to_tsecr) {
7923                                 t = cts - (uint32_t)rsm->r_tim_lastsent[i];
7924                                 if ((int)t <= 0)
7925                                         t = 1;
7926                                 if (CC_ALGO(tp)->rttsample != NULL) {
7927                                         /*
7928                                          * Kick the RTT to the CC, here
7929                                          * we lie a bit in that we know the
7930                                          * retransmission is correct even though
7931                                          * we retransmitted. This is because
7932                                          * we match the timestamps.
7933                                          */
7934                                         if (TSTMP_GT(tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time), rsm->r_tim_lastsent[i]))
7935                                                 us_rtt = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time) - (uint32_t)rsm->r_tim_lastsent[i];
7936                                         else
7937                                                 us_rtt = tcp_get_usecs(NULL) - (uint32_t)rsm->r_tim_lastsent[i];
7938                                         CC_ALGO(tp)->rttsample(tp->ccv, us_rtt, 1, rsm->r_fas);
7939                                 }
7940                                 if ((i + 1) < rsm->r_rtr_cnt) {
7941                                         /*
7942                                          * The peer ack'd from our previous
7943                                          * transmission. We have a spurious
7944                                          * retransmission and thus we dont
7945                                          * want to update our rack_rtt.
7946                                          *
7947                                          * Hmm should there be a CC revert here?
7948                                          *
7949                                          */
7950                                         return (0);
7951                                 }
7952                                 if (!tp->t_rttlow || tp->t_rttlow > t)
7953                                         tp->t_rttlow = t;
7954                                 if (!rack->r_ctl.rc_rack_min_rtt || SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) {
7955                                         rack->r_ctl.rc_rack_min_rtt = t;
7956                                         if (rack->r_ctl.rc_rack_min_rtt == 0) {
7957                                                 rack->r_ctl.rc_rack_min_rtt = 1;
7958                                         }
7959                                 }
7960                                 if (SEQ_LT(rack->r_ctl.rc_rack_tmit_time,
7961                                            (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)])) {
7962                                         /* New more recent rack_tmit_time */
7963                                         rack->r_ctl.rc_rack_tmit_time = (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
7964                                         rack->rc_rack_rtt = t;
7965                                 }
7966                                 rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[i], cts, 3);
7967                                 tcp_rack_xmit_timer(rack, t + 1, len_acked, t, 0, rsm,
7968                                                     rsm->r_rtr_cnt);
7969                                 return (1);
7970                         }
7971                 }
7972                 goto ts_not_found;
7973         } else {
7974                 /*
7975                  * Ok its a SACK block that we retransmitted. or a windows
7976                  * machine without timestamps. We can tell nothing from the
7977                  * time-stamp since its not there or the time the peer last
7978                  * recieved a segment that moved forward its cum-ack point.
7979                  */
7980 ts_not_found:
7981                 i = rsm->r_rtr_cnt - 1;
7982                 t = cts - (uint32_t)rsm->r_tim_lastsent[i];
7983                 if ((int)t <= 0)
7984                         t = 1;
7985                 if (rack->r_ctl.rc_rack_min_rtt && SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) {
7986                         /*
7987                          * We retransmitted and the ack came back in less
7988                          * than the smallest rtt we have observed. We most
7989                          * likely did an improper retransmit as outlined in
7990                          * 6.2 Step 2 point 2 in the rack-draft so we
7991                          * don't want to update our rack_rtt. We in
7992                          * theory (in future) might want to think about reverting our
7993                          * cwnd state but we won't for now.
7994                          */
7995                         return (0);
7996                 } else if (rack->r_ctl.rc_rack_min_rtt) {
7997                         /*
7998                          * We retransmitted it and the retransmit did the
7999                          * job.
8000                          */
8001                         if (!rack->r_ctl.rc_rack_min_rtt ||
8002                             SEQ_LT(t, rack->r_ctl.rc_rack_min_rtt)) {
8003                                 rack->r_ctl.rc_rack_min_rtt = t;
8004                                 if (rack->r_ctl.rc_rack_min_rtt == 0) {
8005                                         rack->r_ctl.rc_rack_min_rtt = 1;
8006                                 }
8007                         }
8008                         if (SEQ_LT(rack->r_ctl.rc_rack_tmit_time, (uint32_t)rsm->r_tim_lastsent[i])) {
8009                                 /* New more recent rack_tmit_time */
8010                                 rack->r_ctl.rc_rack_tmit_time = (uint32_t)rsm->r_tim_lastsent[i];
8011                                 rack->rc_rack_rtt = t;
8012                         }
8013                         return (1);
8014                 }
8015         }
8016         return (0);
8017 }
8018
8019 /*
8020  * Mark the SACK_PASSED flag on all entries prior to rsm send wise.
8021  */
8022 static void
8023 rack_log_sack_passed(struct tcpcb *tp,
8024     struct tcp_rack *rack, struct rack_sendmap *rsm)
8025 {
8026         struct rack_sendmap *nrsm;
8027
8028         nrsm = rsm;
8029         TAILQ_FOREACH_REVERSE_FROM(nrsm, &rack->r_ctl.rc_tmap,
8030             rack_head, r_tnext) {
8031                 if (nrsm == rsm) {
8032                         /* Skip orginal segment he is acked */
8033                         continue;
8034                 }
8035                 if (nrsm->r_flags & RACK_ACKED) {
8036                         /*
8037                          * Skip ack'd segments, though we
8038                          * should not see these, since tmap
8039                          * should not have ack'd segments.
8040                          */
8041                         continue;
8042                 }
8043                 if (nrsm->r_flags & RACK_SACK_PASSED) {
8044                         /*
8045                          * We found one that is already marked
8046                          * passed, we have been here before and
8047                          * so all others below this are marked.
8048                          */
8049                         break;
8050                 }
8051                 nrsm->r_flags |= RACK_SACK_PASSED;
8052                 nrsm->r_flags &= ~RACK_WAS_SACKPASS;
8053         }
8054 }
8055
8056 static void
8057 rack_need_set_test(struct tcpcb *tp,
8058                    struct tcp_rack *rack,
8059                    struct rack_sendmap *rsm,
8060                    tcp_seq th_ack,
8061                    int line,
8062                    int use_which)
8063 {
8064
8065         if ((tp->t_flags & TF_GPUTINPROG) &&
8066             SEQ_GEQ(rsm->r_end, tp->gput_seq)) {
8067                 /*
8068                  * We were app limited, and this ack
8069                  * butts up or goes beyond the point where we want
8070                  * to start our next measurement. We need
8071                  * to record the new gput_ts as here and
8072                  * possibly update the start sequence.
8073                  */
8074                 uint32_t seq, ts;
8075
8076                 if (rsm->r_rtr_cnt > 1) {
8077                         /*
8078                          * This is a retransmit, can we
8079                          * really make any assessment at this
8080                          * point?  We are not really sure of
8081                          * the timestamp, is it this or the
8082                          * previous transmission?
8083                          *
8084                          * Lets wait for something better that
8085                          * is not retransmitted.
8086                          */
8087                         return;
8088                 }
8089                 seq = tp->gput_seq;
8090                 ts = tp->gput_ts;
8091                 rack->app_limited_needs_set = 0;
8092                 tp->gput_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time);
8093                 /* Do we start at a new end? */
8094                 if ((use_which == RACK_USE_BEG) &&
8095                     SEQ_GEQ(rsm->r_start, tp->gput_seq)) {
8096                         /*
8097                          * When we get an ACK that just eats
8098                          * up some of the rsm, we set RACK_USE_BEG
8099                          * since whats at r_start (i.e. th_ack)
8100                          * is left unacked and thats where the
8101                          * measurement not starts.
8102                          */
8103                         tp->gput_seq = rsm->r_start;
8104                         rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
8105                 }
8106                 if ((use_which == RACK_USE_END) &&
8107                     SEQ_GEQ(rsm->r_end, tp->gput_seq)) {
8108                             /*
8109                              * We use the end when the cumack
8110                              * is moving forward and completely
8111                              * deleting the rsm passed so basically
8112                              * r_end holds th_ack.
8113                              *
8114                              * For SACK's we also want to use the end
8115                              * since this piece just got sacked and
8116                              * we want to target anything after that
8117                              * in our measurement.
8118                              */
8119                             tp->gput_seq = rsm->r_end;
8120                             rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
8121                 }
8122                 if (use_which == RACK_USE_END_OR_THACK) {
8123                         /*
8124                          * special case for ack moving forward,
8125                          * not a sack, we need to move all the
8126                          * way up to where this ack cum-ack moves
8127                          * to.
8128                          */
8129                         if (SEQ_GT(th_ack, rsm->r_end))
8130                                 tp->gput_seq = th_ack;
8131                         else
8132                                 tp->gput_seq = rsm->r_end;
8133                         rack->r_ctl.rc_gp_output_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
8134                 }
8135                 if (SEQ_GT(tp->gput_seq, tp->gput_ack)) {
8136                         /*
8137                          * We moved beyond this guy's range, re-calculate
8138                          * the new end point.
8139                          */
8140                         if (rack->rc_gp_filled == 0) {
8141                                 tp->gput_ack = tp->gput_seq + max(rc_init_window(rack), (MIN_GP_WIN * ctf_fixed_maxseg(tp)));
8142                         } else {
8143                                 tp->gput_ack = tp->gput_seq + rack_get_measure_window(tp, rack);
8144                         }
8145                 }
8146                 /*
8147                  * We are moving the goal post, we may be able to clear the
8148                  * measure_saw_probe_rtt flag.
8149                  */
8150                 if ((rack->in_probe_rtt == 0) &&
8151                     (rack->measure_saw_probe_rtt) &&
8152                     (SEQ_GEQ(tp->gput_seq, rack->r_ctl.rc_probertt_sndmax_atexit)))
8153                         rack->measure_saw_probe_rtt = 0;
8154                 rack_log_pacing_delay_calc(rack, ts, tp->gput_ts,
8155                                            seq, tp->gput_seq, 0, 5, line, NULL, 0);
8156                 if (rack->rc_gp_filled &&
8157                     ((tp->gput_ack - tp->gput_seq) <
8158                      max(rc_init_window(rack), (MIN_GP_WIN *
8159                                                 ctf_fixed_maxseg(tp))))) {
8160                         uint32_t ideal_amount;
8161
8162                         ideal_amount = rack_get_measure_window(tp, rack);
8163                         if (ideal_amount > sbavail(&tp->t_inpcb->inp_socket->so_snd)) {
8164                                 /*
8165                                  * There is no sense of continuing this measurement
8166                                  * because its too small to gain us anything we
8167                                  * trust. Skip it and that way we can start a new
8168                                  * measurement quicker.
8169                                  */
8170                                 tp->t_flags &= ~TF_GPUTINPROG;
8171                                 rack_log_pacing_delay_calc(rack, tp->gput_ack, tp->gput_seq,
8172                                                            0, 0, 0, 6, __LINE__, NULL, 0);
8173                         } else {
8174                                 /*
8175                                  * Reset the window further out.
8176                                  */
8177                                 tp->gput_ack = tp->gput_seq + ideal_amount;
8178                         }
8179                 }
8180         }
8181 }
8182
8183 static inline int
8184 is_rsm_inside_declared_tlp_block(struct tcp_rack *rack, struct rack_sendmap *rsm)
8185 {
8186         if (SEQ_LT(rsm->r_end, rack->r_ctl.last_tlp_acked_start)) {
8187                 /* Behind our TLP definition or right at */
8188                 return (0);
8189         }
8190         if (SEQ_GT(rsm->r_start, rack->r_ctl.last_tlp_acked_end)) {
8191                 /* The start is beyond or right at our end of TLP definition */
8192                 return (0);
8193         }
8194         /* It has to be a sub-part of the original TLP recorded */
8195         return (1);
8196 }
8197
8198
8199 static uint32_t
8200 rack_proc_sack_blk(struct tcpcb *tp, struct tcp_rack *rack, struct sackblk *sack,
8201                    struct tcpopt *to, struct rack_sendmap **prsm, uint32_t cts, int *moved_two)
8202 {
8203         uint32_t start, end, changed = 0;
8204         struct rack_sendmap stack_map;
8205         struct rack_sendmap *rsm, *nrsm, fe, *prev, *next;
8206 #ifdef INVARIANTS
8207         struct rack_sendmap *insret;
8208 #endif
8209         int32_t used_ref = 1;
8210         int moved = 0;
8211
8212         start = sack->start;
8213         end = sack->end;
8214         rsm = *prsm;
8215         memset(&fe, 0, sizeof(fe));
8216 do_rest_ofb:
8217         if ((rsm == NULL) ||
8218             (SEQ_LT(end, rsm->r_start)) ||
8219             (SEQ_GEQ(start, rsm->r_end)) ||
8220             (SEQ_LT(start, rsm->r_start))) {
8221                 /*
8222                  * We are not in the right spot,
8223                  * find the correct spot in the tree.
8224                  */
8225                 used_ref = 0;
8226                 fe.r_start = start;
8227                 rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe);
8228                 moved++;
8229         }
8230         if (rsm == NULL) {
8231                 /* TSNH */
8232                 goto out;
8233         }
8234         /* Ok we have an ACK for some piece of this rsm */
8235         if (rsm->r_start != start) {
8236                 if ((rsm->r_flags & RACK_ACKED) == 0) {
8237                         /*
8238                          * Before any splitting or hookery is
8239                          * done is it a TLP of interest i.e. rxt?
8240                          */
8241                         if ((rsm->r_flags & RACK_TLP) &&
8242                             (rsm->r_rtr_cnt > 1)) {
8243                                 /*
8244                                  * We are splitting a rxt TLP, check
8245                                  * if we need to save off the start/end
8246                                  */
8247                                 if (rack->rc_last_tlp_acked_set &&
8248                                     (is_rsm_inside_declared_tlp_block(rack, rsm))) {
8249                                         /*
8250                                          * We already turned this on since we are inside
8251                                          * the previous one was a partially sack now we
8252                                          * are getting another one (maybe all of it).
8253                                          *
8254                                          */
8255                                         rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
8256                                         /*
8257                                          * Lets make sure we have all of it though.
8258                                          */
8259                                         if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
8260                                                 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
8261                                                 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
8262                                                                      rack->r_ctl.last_tlp_acked_end);
8263                                         }
8264                                         if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
8265                                                 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
8266                                                 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
8267                                                                      rack->r_ctl.last_tlp_acked_end);
8268                                         }
8269                                 } else {
8270                                         rack->r_ctl.last_tlp_acked_start = rsm->r_start;
8271                                         rack->r_ctl.last_tlp_acked_end = rsm->r_end;
8272                                         rack->rc_last_tlp_past_cumack = 0;
8273                                         rack->rc_last_tlp_acked_set = 1;
8274                                         rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
8275                                 }
8276                         }
8277                         /**
8278                          * Need to split this in two pieces the before and after,
8279                          * the before remains in the map, the after must be
8280                          * added. In other words we have:
8281                          * rsm        |--------------|
8282                          * sackblk        |------->
8283                          * rsm will become
8284                          *     rsm    |---|
8285                          * and nrsm will be  the sacked piece
8286                          *     nrsm       |----------|
8287                          *
8288                          * But before we start down that path lets
8289                          * see if the sack spans over on top of
8290                          * the next guy and it is already sacked.
8291                          *
8292                          */
8293                         next = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8294                         if (next && (next->r_flags & RACK_ACKED) &&
8295                             SEQ_GEQ(end, next->r_start)) {
8296                                 /**
8297                                  * So the next one is already acked, and
8298                                  * we can thus by hookery use our stack_map
8299                                  * to reflect the piece being sacked and
8300                                  * then adjust the two tree entries moving
8301                                  * the start and ends around. So we start like:
8302                                  *  rsm     |------------|             (not-acked)
8303                                  *  next                 |-----------| (acked)
8304                                  *  sackblk        |-------->
8305                                  *  We want to end like so:
8306                                  *  rsm     |------|                   (not-acked)
8307                                  *  next           |-----------------| (acked)
8308                                  *  nrsm           |-----|
8309                                  * Where nrsm is a temporary stack piece we
8310                                  * use to update all the gizmos.
8311                                  */
8312                                 /* Copy up our fudge block */
8313                                 nrsm = &stack_map;
8314                                 memcpy(nrsm, rsm, sizeof(struct rack_sendmap));
8315                                 /* Now adjust our tree blocks */
8316                                 rsm->r_end = start;
8317                                 next->r_start = start;
8318                                 /* Now we must adjust back where next->m is */
8319                                 rack_setup_offset_for_rsm(rsm, next);
8320
8321                                 /* We don't need to adjust rsm, it did not change */
8322                                 /* Clear out the dup ack count of the remainder */
8323                                 rsm->r_dupack = 0;
8324                                 rsm->r_just_ret = 0;
8325                                 rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
8326                                 /* Now lets make sure our fudge block is right */
8327                                 nrsm->r_start = start;
8328                                 /* Now lets update all the stats and such */
8329                                 rack_update_rtt(tp, rack, nrsm, to, cts, SACKED, 0);
8330                                 if (rack->app_limited_needs_set)
8331                                         rack_need_set_test(tp, rack, nrsm, tp->snd_una, __LINE__, RACK_USE_END);
8332                                 changed += (nrsm->r_end - nrsm->r_start);
8333                                 rack->r_ctl.rc_sacked += (nrsm->r_end - nrsm->r_start);
8334                                 if (nrsm->r_flags & RACK_SACK_PASSED) {
8335                                         rack->r_ctl.rc_reorder_ts = cts;
8336                                 }
8337                                 /*
8338                                  * Now we want to go up from rsm (the
8339                                  * one left un-acked) to the next one
8340                                  * in the tmap. We do this so when
8341                                  * we walk backwards we include marking
8342                                  * sack-passed on rsm (The one passed in
8343                                  * is skipped since it is generally called
8344                                  * on something sacked before removing it
8345                                  * from the tmap).
8346                                  */
8347                                 if (rsm->r_in_tmap) {
8348                                         nrsm = TAILQ_NEXT(rsm, r_tnext);
8349                                         /*
8350                                          * Now that we have the next
8351                                          * one walk backwards from there.
8352                                          */
8353                                         if (nrsm && nrsm->r_in_tmap)
8354                                                 rack_log_sack_passed(tp, rack, nrsm);
8355                                 }
8356                                 /* Now are we done? */
8357                                 if (SEQ_LT(end, next->r_end) ||
8358                                     (end == next->r_end)) {
8359                                         /* Done with block */
8360                                         goto out;
8361                                 }
8362                                 rack_log_map_chg(tp, rack, &stack_map, rsm, next, MAP_SACK_M1, end, __LINE__);
8363                                 counter_u64_add(rack_sack_used_next_merge, 1);
8364                                 /* Postion for the next block */
8365                                 start = next->r_end;
8366                                 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, next);
8367                                 if (rsm == NULL)
8368                                         goto out;
8369                         } else {
8370                                 /**
8371                                  * We can't use any hookery here, so we
8372                                  * need to split the map. We enter like
8373                                  * so:
8374                                  *  rsm      |--------|
8375                                  *  sackblk       |----->
8376                                  * We will add the new block nrsm and
8377                                  * that will be the new portion, and then
8378                                  * fall through after reseting rsm. So we
8379                                  * split and look like this:
8380                                  *  rsm      |----|
8381                                  *  sackblk       |----->
8382                                  *  nrsm          |---|
8383                                  * We then fall through reseting
8384                                  * rsm to nrsm, so the next block
8385                                  * picks it up.
8386                                  */
8387                                 nrsm = rack_alloc_limit(rack, RACK_LIMIT_TYPE_SPLIT);
8388                                 if (nrsm == NULL) {
8389                                         /*
8390                                          * failed XXXrrs what can we do but loose the sack
8391                                          * info?
8392                                          */
8393                                         goto out;
8394                                 }
8395                                 counter_u64_add(rack_sack_splits, 1);
8396                                 rack_clone_rsm(rack, nrsm, rsm, start);
8397                                 rsm->r_just_ret = 0;
8398 #ifndef INVARIANTS
8399                                 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
8400 #else
8401                                 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
8402                                 if (insret != NULL) {
8403                                         panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p",
8404                                               nrsm, insret, rack, rsm);
8405                                 }
8406 #endif
8407                                 if (rsm->r_in_tmap) {
8408                                         TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8409                                         nrsm->r_in_tmap = 1;
8410                                 }
8411                                 rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SACK_M2, end, __LINE__);
8412                                 rsm->r_flags &= (~RACK_HAS_FIN);
8413                                 /* Position us to point to the new nrsm that starts the sack blk */
8414                                 rsm = nrsm;
8415                         }
8416                 } else {
8417                         /* Already sacked this piece */
8418                         counter_u64_add(rack_sack_skipped_acked, 1);
8419                         moved++;
8420                         if (end == rsm->r_end) {
8421                                 /* Done with block */
8422                                 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8423                                 goto out;
8424                         } else if (SEQ_LT(end, rsm->r_end)) {
8425                                 /* A partial sack to a already sacked block */
8426                                 moved++;
8427                                 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8428                                 goto out;
8429                         } else {
8430                                 /*
8431                                  * The end goes beyond this guy
8432                                  * reposition the start to the
8433                                  * next block.
8434                                  */
8435                                 start = rsm->r_end;
8436                                 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8437                                 if (rsm == NULL)
8438                                         goto out;
8439                         }
8440                 }
8441         }
8442         if (SEQ_GEQ(end, rsm->r_end)) {
8443                 /**
8444                  * The end of this block is either beyond this guy or right
8445                  * at this guy. I.e.:
8446                  *  rsm ---                 |-----|
8447                  *  end                     |-----|
8448                  *  <or>
8449                  *  end                     |---------|
8450                  */
8451                 if ((rsm->r_flags & RACK_ACKED) == 0) {
8452                         /*
8453                          * Is it a TLP of interest?
8454                          */
8455                         if ((rsm->r_flags & RACK_TLP) &&
8456                             (rsm->r_rtr_cnt > 1)) {
8457                                 /*
8458                                  * We are splitting a rxt TLP, check
8459                                  * if we need to save off the start/end
8460                                  */
8461                                 if (rack->rc_last_tlp_acked_set &&
8462                                     (is_rsm_inside_declared_tlp_block(rack, rsm))) {
8463                                         /*
8464                                          * We already turned this on since we are inside
8465                                          * the previous one was a partially sack now we
8466                                          * are getting another one (maybe all of it).
8467                                          */
8468                                         rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
8469                                         /*
8470                                          * Lets make sure we have all of it though.
8471                                          */
8472                                         if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
8473                                                 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
8474                                                 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
8475                                                                      rack->r_ctl.last_tlp_acked_end);
8476                                         }
8477                                         if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
8478                                                 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
8479                                                 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
8480                                                                      rack->r_ctl.last_tlp_acked_end);
8481                                         }
8482                                 } else {
8483                                         rack->r_ctl.last_tlp_acked_start = rsm->r_start;
8484                                         rack->r_ctl.last_tlp_acked_end = rsm->r_end;
8485                                         rack->rc_last_tlp_past_cumack = 0;
8486                                         rack->rc_last_tlp_acked_set = 1;
8487                                         rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
8488                                 }
8489                         }
8490                         rack_update_rtt(tp, rack, rsm, to, cts, SACKED, 0);
8491                         changed += (rsm->r_end - rsm->r_start);
8492                         rack->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
8493                         if (rsm->r_in_tmap) /* should be true */
8494                                 rack_log_sack_passed(tp, rack, rsm);
8495                         /* Is Reordering occuring? */
8496                         if (rsm->r_flags & RACK_SACK_PASSED) {
8497                                 rsm->r_flags &= ~RACK_SACK_PASSED;
8498                                 rack->r_ctl.rc_reorder_ts = cts;
8499                         }
8500                         if (rack->app_limited_needs_set)
8501                                 rack_need_set_test(tp, rack, rsm, tp->snd_una, __LINE__, RACK_USE_END);
8502                         rsm->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
8503                         rsm->r_flags |= RACK_ACKED;
8504                         if (rsm->r_in_tmap) {
8505                                 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
8506                                 rsm->r_in_tmap = 0;
8507                         }
8508                         rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_SACK_M3, end, __LINE__);
8509                 } else {
8510                         counter_u64_add(rack_sack_skipped_acked, 1);
8511                         moved++;
8512                 }
8513                 if (end == rsm->r_end) {
8514                         /* This block only - done, setup for next */
8515                         goto out;
8516                 }
8517                 /*
8518                  * There is more not coverend by this rsm move on
8519                  * to the next block in the RB tree.
8520                  */
8521                 nrsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8522                 start = rsm->r_end;
8523                 rsm = nrsm;
8524                 if (rsm == NULL)
8525                         goto out;
8526                 goto do_rest_ofb;
8527         }
8528         /**
8529          * The end of this sack block is smaller than
8530          * our rsm i.e.:
8531          *  rsm ---                 |-----|
8532          *  end                     |--|
8533          */
8534         if ((rsm->r_flags & RACK_ACKED) == 0) {
8535                 /*
8536                  * Is it a TLP of interest?
8537                  */
8538                 if ((rsm->r_flags & RACK_TLP) &&
8539                     (rsm->r_rtr_cnt > 1)) {
8540                         /*
8541                          * We are splitting a rxt TLP, check
8542                          * if we need to save off the start/end
8543                          */
8544                         if (rack->rc_last_tlp_acked_set &&
8545                             (is_rsm_inside_declared_tlp_block(rack, rsm))) {
8546                                 /*
8547                                  * We already turned this on since we are inside
8548                                  * the previous one was a partially sack now we
8549                                  * are getting another one (maybe all of it).
8550                                  */
8551                                 rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
8552                                 /*
8553                                  * Lets make sure we have all of it though.
8554                                  */
8555                                 if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
8556                                         rack->r_ctl.last_tlp_acked_start = rsm->r_start;
8557                                         rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
8558                                                              rack->r_ctl.last_tlp_acked_end);
8559                                 }
8560                                 if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
8561                                         rack->r_ctl.last_tlp_acked_end = rsm->r_end;
8562                                         rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
8563                                                              rack->r_ctl.last_tlp_acked_end);
8564                                 }
8565                         } else {
8566                                 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
8567                                 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
8568                                 rack->rc_last_tlp_past_cumack = 0;
8569                                 rack->rc_last_tlp_acked_set = 1;
8570                                 rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
8571                         }
8572                 }
8573                 prev = RB_PREV(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8574                 if (prev &&
8575                     (prev->r_flags & RACK_ACKED)) {
8576                         /**
8577                          * Goal, we want the right remainder of rsm to shrink
8578                          * in place and span from (rsm->r_start = end) to rsm->r_end.
8579                          * We want to expand prev to go all the way
8580                          * to prev->r_end <- end.
8581                          * so in the tree we have before:
8582                          *   prev     |--------|         (acked)
8583                          *   rsm               |-------| (non-acked)
8584                          *   sackblk           |-|
8585                          * We churn it so we end up with
8586                          *   prev     |----------|       (acked)
8587                          *   rsm                 |-----| (non-acked)
8588                          *   nrsm              |-| (temporary)
8589                          *
8590                          * Note if either prev/rsm is a TLP we don't
8591                          * do this.
8592                          */
8593                         nrsm = &stack_map;
8594                         memcpy(nrsm, rsm, sizeof(struct rack_sendmap));
8595                         prev->r_end = end;
8596                         rsm->r_start = end;
8597                         /* Now adjust nrsm (stack copy) to be
8598                          * the one that is the small
8599                          * piece that was "sacked".
8600                          */
8601                         nrsm->r_end = end;
8602                         rsm->r_dupack = 0;
8603                         rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
8604                         /*
8605                          * Now that the rsm has had its start moved forward
8606                          * lets go ahead and get its new place in the world.
8607                          */
8608                         rack_setup_offset_for_rsm(prev, rsm);
8609                         /*
8610                          * Now nrsm is our new little piece
8611                          * that is acked (which was merged
8612                          * to prev). Update the rtt and changed
8613                          * based on that. Also check for reordering.
8614                          */
8615                         rack_update_rtt(tp, rack, nrsm, to, cts, SACKED, 0);
8616                         if (rack->app_limited_needs_set)
8617                                 rack_need_set_test(tp, rack, nrsm, tp->snd_una, __LINE__, RACK_USE_END);
8618                         changed += (nrsm->r_end - nrsm->r_start);
8619                         rack->r_ctl.rc_sacked += (nrsm->r_end - nrsm->r_start);
8620                         if (nrsm->r_flags & RACK_SACK_PASSED) {
8621                                 rack->r_ctl.rc_reorder_ts = cts;
8622                         }
8623                         rack_log_map_chg(tp, rack, prev, &stack_map, rsm, MAP_SACK_M4, end, __LINE__);
8624                         rsm = prev;
8625                         counter_u64_add(rack_sack_used_prev_merge, 1);
8626                 } else {
8627                         /**
8628                          * This is the case where our previous
8629                          * block is not acked either, so we must
8630                          * split the block in two.
8631                          */
8632                         nrsm = rack_alloc_limit(rack, RACK_LIMIT_TYPE_SPLIT);
8633                         if (nrsm == NULL) {
8634                                 /* failed rrs what can we do but loose the sack info? */
8635                                 goto out;
8636                         }
8637                         if ((rsm->r_flags & RACK_TLP) &&
8638                             (rsm->r_rtr_cnt > 1)) {
8639                                 /*
8640                                  * We are splitting a rxt TLP, check
8641                                  * if we need to save off the start/end
8642                                  */
8643                                 if (rack->rc_last_tlp_acked_set &&
8644                                     (is_rsm_inside_declared_tlp_block(rack, rsm))) {
8645                                             /*
8646                                              * We already turned this on since this block is inside
8647                                              * the previous one was a partially sack now we
8648                                              * are getting another one (maybe all of it).
8649                                              */
8650                                             rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
8651                                             /*
8652                                              * Lets make sure we have all of it though.
8653                                              */
8654                                             if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
8655                                                     rack->r_ctl.last_tlp_acked_start = rsm->r_start;
8656                                                     rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
8657                                                                          rack->r_ctl.last_tlp_acked_end);
8658                                             }
8659                                             if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
8660                                                     rack->r_ctl.last_tlp_acked_end = rsm->r_end;
8661                                                     rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
8662                                                                          rack->r_ctl.last_tlp_acked_end);
8663                                             }
8664                                     } else {
8665                                             rack->r_ctl.last_tlp_acked_start = rsm->r_start;
8666                                             rack->r_ctl.last_tlp_acked_end = rsm->r_end;
8667                                             rack->rc_last_tlp_acked_set = 1;
8668                                             rack->rc_last_tlp_past_cumack = 0;
8669                                             rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
8670                                     }
8671                         }
8672                         /**
8673                          * In this case nrsm becomes
8674                          * nrsm->r_start = end;
8675                          * nrsm->r_end = rsm->r_end;
8676                          * which is un-acked.
8677                          * <and>
8678                          * rsm->r_end = nrsm->r_start;
8679                          * i.e. the remaining un-acked
8680                          * piece is left on the left
8681                          * hand side.
8682                          *
8683                          * So we start like this
8684                          * rsm      |----------| (not acked)
8685                          * sackblk  |---|
8686                          * build it so we have
8687                          * rsm      |---|         (acked)
8688                          * nrsm         |------|  (not acked)
8689                          */
8690                         counter_u64_add(rack_sack_splits, 1);
8691                         rack_clone_rsm(rack, nrsm, rsm, end);
8692                         rsm->r_flags &= (~RACK_HAS_FIN);
8693                         rsm->r_just_ret = 0;
8694 #ifndef INVARIANTS
8695                         (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
8696 #else
8697                         insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
8698                         if (insret != NULL) {
8699                                 panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p",
8700                                       nrsm, insret, rack, rsm);
8701                         }
8702 #endif
8703                         if (rsm->r_in_tmap) {
8704                                 TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8705                                 nrsm->r_in_tmap = 1;
8706                         }
8707                         nrsm->r_dupack = 0;
8708                         rack_log_retran_reason(rack, nrsm, __LINE__, 0, 2);
8709                         rack_update_rtt(tp, rack, rsm, to, cts, SACKED, 0);
8710                         changed += (rsm->r_end - rsm->r_start);
8711                         rack->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
8712                         if (rsm->r_in_tmap) /* should be true */
8713                                 rack_log_sack_passed(tp, rack, rsm);
8714                         /* Is Reordering occuring? */
8715                         if (rsm->r_flags & RACK_SACK_PASSED) {
8716                                 rsm->r_flags &= ~RACK_SACK_PASSED;
8717                                 rack->r_ctl.rc_reorder_ts = cts;
8718                         }
8719                         if (rack->app_limited_needs_set)
8720                                 rack_need_set_test(tp, rack, rsm, tp->snd_una, __LINE__, RACK_USE_END);
8721                         rsm->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
8722                         rsm->r_flags |= RACK_ACKED;
8723                         rack_log_map_chg(tp, rack, NULL, rsm, nrsm, MAP_SACK_M5, end, __LINE__);
8724                         if (rsm->r_in_tmap) {
8725                                 TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
8726                                 rsm->r_in_tmap = 0;
8727                         }
8728                 }
8729         } else if (start != end){
8730                 /*
8731                  * The block was already acked.
8732                  */
8733                 counter_u64_add(rack_sack_skipped_acked, 1);
8734                 moved++;
8735         }
8736 out:
8737         if (rsm &&
8738             ((rsm->r_flags & RACK_TLP) == 0) &&
8739             (rsm->r_flags & RACK_ACKED)) {
8740                 /*
8741                  * Now can we merge where we worked
8742                  * with either the previous or
8743                  * next block?
8744                  */
8745                 next = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8746                 while (next) {
8747                         if (next->r_flags & RACK_TLP)
8748                                 break;
8749                         if (next->r_flags & RACK_ACKED) {
8750                         /* yep this and next can be merged */
8751                                 rsm = rack_merge_rsm(rack, rsm, next);
8752                                 next = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8753                         } else
8754                                 break;
8755                 }
8756                 /* Now what about the previous? */
8757                 prev = RB_PREV(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8758                 while (prev) {
8759                         if (prev->r_flags & RACK_TLP)
8760                                 break;
8761                         if (prev->r_flags & RACK_ACKED) {
8762                                 /* yep the previous and this can be merged */
8763                                 rsm = rack_merge_rsm(rack, prev, rsm);
8764                                 prev = RB_PREV(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8765                         } else
8766                                 break;
8767                 }
8768         }
8769         if (used_ref == 0) {
8770                 counter_u64_add(rack_sack_proc_all, 1);
8771         } else {
8772                 counter_u64_add(rack_sack_proc_short, 1);
8773         }
8774         /* Save off the next one for quick reference. */
8775         if (rsm)
8776                 nrsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8777         else
8778                 nrsm = NULL;
8779         *prsm = rack->r_ctl.rc_sacklast = nrsm;
8780         /* Pass back the moved. */
8781         *moved_two = moved;
8782         return (changed);
8783 }
8784
8785 static void inline
8786 rack_peer_reneges(struct tcp_rack *rack, struct rack_sendmap *rsm, tcp_seq th_ack)
8787 {
8788         struct rack_sendmap *tmap;
8789
8790         tmap = NULL;
8791         while (rsm && (rsm->r_flags & RACK_ACKED)) {
8792                 /* Its no longer sacked, mark it so */
8793                 rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
8794 #ifdef INVARIANTS
8795                 if (rsm->r_in_tmap) {
8796                         panic("rack:%p rsm:%p flags:0x%x in tmap?",
8797                               rack, rsm, rsm->r_flags);
8798                 }
8799 #endif
8800                 rsm->r_flags &= ~(RACK_ACKED|RACK_SACK_PASSED|RACK_WAS_SACKPASS);
8801                 /* Rebuild it into our tmap */
8802                 if (tmap == NULL) {
8803                         TAILQ_INSERT_HEAD(&rack->r_ctl.rc_tmap, rsm, r_tnext);
8804                         tmap = rsm;
8805                 } else {
8806                         TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, tmap, rsm, r_tnext);
8807                         tmap = rsm;
8808                 }
8809                 tmap->r_in_tmap = 1;
8810                 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
8811         }
8812         /*
8813          * Now lets possibly clear the sack filter so we start
8814          * recognizing sacks that cover this area.
8815          */
8816         sack_filter_clear(&rack->r_ctl.rack_sf, th_ack);
8817
8818 }
8819
8820 static void
8821 rack_do_decay(struct tcp_rack *rack)
8822 {
8823         struct timeval res;
8824
8825 #define timersub(tvp, uvp, vvp)                                         \
8826         do {                                                            \
8827                 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec;          \
8828                 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec;       \
8829                 if ((vvp)->tv_usec < 0) {                               \
8830                         (vvp)->tv_sec--;                                \
8831                         (vvp)->tv_usec += 1000000;                      \
8832                 }                                                       \
8833         } while (0)
8834
8835         timersub(&rack->r_ctl.act_rcv_time, &rack->r_ctl.rc_last_time_decay, &res);
8836 #undef timersub
8837
8838         rack->r_ctl.input_pkt++;
8839         if ((rack->rc_in_persist) ||
8840             (res.tv_sec >= 1) ||
8841             (rack->rc_tp->snd_max == rack->rc_tp->snd_una)) {
8842                 /*
8843                  * Check for decay of non-SAD,
8844                  * we want all SAD detection metrics to
8845                  * decay 1/4 per second (or more) passed.
8846                  */
8847 #ifdef NETFLIX_EXP_DETECTION
8848                 uint32_t pkt_delta;
8849
8850                 pkt_delta = rack->r_ctl.input_pkt - rack->r_ctl.saved_input_pkt;
8851 #endif
8852                 /* Update our saved tracking values */
8853                 rack->r_ctl.saved_input_pkt = rack->r_ctl.input_pkt;
8854                 rack->r_ctl.rc_last_time_decay = rack->r_ctl.act_rcv_time;
8855                 /* Now do we escape without decay? */
8856 #ifdef NETFLIX_EXP_DETECTION
8857                 if (rack->rc_in_persist ||
8858                     (rack->rc_tp->snd_max == rack->rc_tp->snd_una) ||
8859                     (pkt_delta < tcp_sad_low_pps)){
8860                         /*
8861                          * We don't decay idle connections
8862                          * or ones that have a low input pps.
8863                          */
8864                         return;
8865                 }
8866                 /* Decay the counters */
8867                 rack->r_ctl.ack_count = ctf_decay_count(rack->r_ctl.ack_count,
8868                                                         tcp_sad_decay_val);
8869                 rack->r_ctl.sack_count = ctf_decay_count(rack->r_ctl.sack_count,
8870                                                          tcp_sad_decay_val);
8871                 rack->r_ctl.sack_moved_extra = ctf_decay_count(rack->r_ctl.sack_moved_extra,
8872                                                                tcp_sad_decay_val);
8873                 rack->r_ctl.sack_noextra_move = ctf_decay_count(rack->r_ctl.sack_noextra_move,
8874                                                                 tcp_sad_decay_val);
8875 #endif
8876         }
8877 }
8878
8879 static void
8880 rack_process_to_cumack(struct tcpcb *tp, struct tcp_rack *rack, register uint32_t th_ack, uint32_t cts, struct tcpopt *to)
8881 {
8882         struct rack_sendmap *rsm;
8883 #ifdef INVARIANTS
8884         struct rack_sendmap *rm;
8885 #endif
8886
8887         /*
8888          * The ACK point is advancing to th_ack, we must drop off
8889          * the packets in the rack log and calculate any eligble
8890          * RTT's.
8891          */
8892         rack->r_wanted_output = 1;
8893
8894         /* Tend any TLP that has been marked for 1/2 the seq space (its old)  */
8895         if ((rack->rc_last_tlp_acked_set == 1)&&
8896             (rack->rc_last_tlp_past_cumack == 1) &&
8897             (SEQ_GT(rack->r_ctl.last_tlp_acked_start, th_ack))) {
8898                 /*
8899                  * We have reached the point where our last rack
8900                  * tlp retransmit sequence is ahead of the cum-ack.
8901                  * This can only happen when the cum-ack moves all
8902                  * the way around (its been a full 2^^31+1 bytes
8903                  * or more since we sent a retransmitted TLP). Lets
8904                  * turn off the valid flag since its not really valid.
8905                  *
8906                  * Note since sack's also turn on this event we have
8907                  * a complication, we have to wait to age it out until
8908                  * the cum-ack is by the TLP before checking which is
8909                  * what the next else clause does.
8910                  */
8911                 rack_log_dsack_event(rack, 9, __LINE__,
8912                                      rack->r_ctl.last_tlp_acked_start,
8913                                      rack->r_ctl.last_tlp_acked_end);
8914                 rack->rc_last_tlp_acked_set = 0;
8915                 rack->rc_last_tlp_past_cumack = 0;
8916         } else if ((rack->rc_last_tlp_acked_set == 1) &&
8917                    (rack->rc_last_tlp_past_cumack == 0) &&
8918                    (SEQ_GEQ(th_ack, rack->r_ctl.last_tlp_acked_end))) {
8919                 /*
8920                  * It is safe to start aging TLP's out.
8921                  */
8922                 rack->rc_last_tlp_past_cumack = 1;
8923         }
8924         /* We do the same for the tlp send seq as well */
8925         if ((rack->rc_last_sent_tlp_seq_valid == 1) &&
8926             (rack->rc_last_sent_tlp_past_cumack == 1) &&
8927             (SEQ_GT(rack->r_ctl.last_sent_tlp_seq,  th_ack))) {
8928                 rack_log_dsack_event(rack, 9, __LINE__,
8929                                      rack->r_ctl.last_sent_tlp_seq,
8930                                      (rack->r_ctl.last_sent_tlp_seq +
8931                                       rack->r_ctl.last_sent_tlp_len));
8932                 rack->rc_last_sent_tlp_seq_valid = 0;
8933                 rack->rc_last_sent_tlp_past_cumack = 0;
8934         } else if ((rack->rc_last_sent_tlp_seq_valid == 1) &&
8935                    (rack->rc_last_sent_tlp_past_cumack == 0) &&
8936                    (SEQ_GEQ(th_ack, rack->r_ctl.last_sent_tlp_seq))) {
8937                 /*
8938                  * It is safe to start aging TLP's send.
8939                  */
8940                 rack->rc_last_sent_tlp_past_cumack = 1;
8941         }
8942 more:
8943         rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
8944         if (rsm == NULL) {
8945                 if ((th_ack - 1) == tp->iss) {
8946                         /*
8947                          * For the SYN incoming case we will not
8948                          * have called tcp_output for the sending of
8949                          * the SYN, so there will be no map. All
8950                          * other cases should probably be a panic.
8951                          */
8952                         return;
8953                 }
8954                 if (tp->t_flags & TF_SENTFIN) {
8955                         /* if we sent a FIN we often will not have map */
8956                         return;
8957                 }
8958 #ifdef INVARIANTS
8959                 panic("No rack map tp:%p for state:%d ack:%u rack:%p snd_una:%u snd_max:%u snd_nxt:%u\n",
8960                       tp,
8961                       tp->t_state, th_ack, rack,
8962                       tp->snd_una, tp->snd_max, tp->snd_nxt);
8963 #endif
8964                 return;
8965         }
8966         if (SEQ_LT(th_ack, rsm->r_start)) {
8967                 /* Huh map is missing this */
8968 #ifdef INVARIANTS
8969                 printf("Rack map starts at r_start:%u for th_ack:%u huh? ts:%d rs:%d\n",
8970                        rsm->r_start,
8971                        th_ack, tp->t_state, rack->r_state);
8972 #endif
8973                 return;
8974         }
8975         rack_update_rtt(tp, rack, rsm, to, cts, CUM_ACKED, th_ack);
8976
8977         /* Now was it a retransmitted TLP? */
8978         if ((rsm->r_flags & RACK_TLP) &&
8979             (rsm->r_rtr_cnt > 1)) {
8980                 /*
8981                  * Yes, this rsm was a TLP and retransmitted, remember that
8982                  * since if a DSACK comes back on this we don't want
8983                  * to think of it as a reordered segment. This may
8984                  * get updated again with possibly even other TLPs
8985                  * in flight, but thats ok. Only when we don't send
8986                  * a retransmitted TLP for 1/2 the sequences space
8987                  * will it get turned off (above).
8988                  */
8989                 if (rack->rc_last_tlp_acked_set &&
8990                     (is_rsm_inside_declared_tlp_block(rack, rsm))) {
8991                         /*
8992                          * We already turned this on since the end matches,
8993                          * the previous one was a partially ack now we
8994                          * are getting another one (maybe all of it).
8995                          */
8996                         rack_log_dsack_event(rack, 10, __LINE__, rsm->r_start, rsm->r_end);
8997                         /*
8998                          * Lets make sure we have all of it though.
8999                          */
9000                         if (SEQ_LT(rsm->r_start, rack->r_ctl.last_tlp_acked_start)) {
9001                                 rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9002                                 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9003                                                      rack->r_ctl.last_tlp_acked_end);
9004                         }
9005                         if (SEQ_GT(rsm->r_end, rack->r_ctl.last_tlp_acked_end)) {
9006                                 rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9007                                 rack_log_dsack_event(rack, 11, __LINE__, rack->r_ctl.last_tlp_acked_start,
9008                                                      rack->r_ctl.last_tlp_acked_end);
9009                         }
9010                 } else {
9011                         rack->rc_last_tlp_past_cumack = 1;
9012                         rack->r_ctl.last_tlp_acked_start = rsm->r_start;
9013                         rack->r_ctl.last_tlp_acked_end = rsm->r_end;
9014                         rack->rc_last_tlp_acked_set = 1;
9015                         rack_log_dsack_event(rack, 8, __LINE__, rsm->r_start, rsm->r_end);
9016                 }
9017         }
9018         /* Now do we consume the whole thing? */
9019         if (SEQ_GEQ(th_ack, rsm->r_end)) {
9020                 /* Its all consumed. */
9021                 uint32_t left;
9022                 uint8_t newly_acked;
9023
9024                 rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_FREE, rsm->r_end, __LINE__);
9025                 rack->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
9026                 rsm->r_rtr_bytes = 0;
9027                 /* Record the time of highest cumack sent */
9028                 rack->r_ctl.rc_gp_cumack_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
9029 #ifndef INVARIANTS
9030                 (void)RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
9031 #else
9032                 rm = RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
9033                 if (rm != rsm) {
9034                         panic("removing head in rack:%p rsm:%p rm:%p",
9035                               rack, rsm, rm);
9036                 }
9037 #endif
9038                 if (rsm->r_in_tmap) {
9039                         TAILQ_REMOVE(&rack->r_ctl.rc_tmap, rsm, r_tnext);
9040                         rsm->r_in_tmap = 0;
9041                 }
9042                 newly_acked = 1;
9043                 if (rsm->r_flags & RACK_ACKED) {
9044                         /*
9045                          * It was acked on the scoreboard -- remove
9046                          * it from total
9047                          */
9048                         rack->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
9049                         newly_acked = 0;
9050                 } else if (rsm->r_flags & RACK_SACK_PASSED) {
9051                         /*
9052                          * There are segments ACKED on the
9053                          * scoreboard further up. We are seeing
9054                          * reordering.
9055                          */
9056                         rsm->r_flags &= ~RACK_SACK_PASSED;
9057                         rsm->r_ack_arrival = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
9058                         rsm->r_flags |= RACK_ACKED;
9059                         rack->r_ctl.rc_reorder_ts = cts;
9060                         if (rack->r_ent_rec_ns) {
9061                                 /*
9062                                  * We have sent no more, and we saw an sack
9063                                  * then ack arrive.
9064                                  */
9065                                 rack->r_might_revert = 1;
9066                         }
9067                 }
9068                 if ((rsm->r_flags & RACK_TO_REXT) &&
9069                     (tp->t_flags & TF_RCVD_TSTMP) &&
9070                     (to->to_flags & TOF_TS) &&
9071                     (to->to_tsecr != 0) &&
9072                     (tp->t_flags & TF_PREVVALID)) {
9073                         /*
9074                          * We can use the timestamp to see
9075                          * if this retransmission was from the
9076                          * first transmit. If so we made a mistake.
9077                          */
9078                         tp->t_flags &= ~TF_PREVVALID;
9079                         if (to->to_tsecr == rack_ts_to_msec(rsm->r_tim_lastsent[0])) {
9080                                 /* The first transmit is what this ack is for */
9081                                 rack_cong_signal(tp, CC_RTO_ERR, th_ack, __LINE__);
9082                         }
9083                 }
9084                 left = th_ack - rsm->r_end;
9085                 if (rack->app_limited_needs_set && newly_acked)
9086                         rack_need_set_test(tp, rack, rsm, th_ack, __LINE__, RACK_USE_END_OR_THACK);
9087                 /* Free back to zone */
9088                 rack_free(rack, rsm);
9089                 if (left) {
9090                         goto more;
9091                 }
9092                 /* Check for reneging */
9093                 rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
9094                 if (rsm && (rsm->r_flags & RACK_ACKED) && (th_ack == rsm->r_start)) {
9095                         /*
9096                          * The peer has moved snd_una up to
9097                          * the edge of this send, i.e. one
9098                          * that it had previously acked. The only
9099                          * way that can be true if the peer threw
9100                          * away data (space issues) that it had
9101                          * previously sacked (else it would have
9102                          * given us snd_una up to (rsm->r_end).
9103                          * We need to undo the acked markings here.
9104                          *
9105                          * Note we have to look to make sure th_ack is
9106                          * our rsm->r_start in case we get an old ack
9107                          * where th_ack is behind snd_una.
9108                          */
9109                         rack_peer_reneges(rack, rsm, th_ack);
9110                 }
9111                 return;
9112         }
9113         if (rsm->r_flags & RACK_ACKED) {
9114                 /*
9115                  * It was acked on the scoreboard -- remove it from
9116                  * total for the part being cum-acked.
9117                  */
9118                 rack->r_ctl.rc_sacked -= (th_ack - rsm->r_start);
9119         }
9120         /*
9121          * Clear the dup ack count for
9122          * the piece that remains.
9123          */
9124         rsm->r_dupack = 0;
9125         rack_log_retran_reason(rack, rsm, __LINE__, 0, 2);
9126         if (rsm->r_rtr_bytes) {
9127                 /*
9128                  * It was retransmitted adjust the
9129                  * sack holes for what was acked.
9130                  */
9131                 int ack_am;
9132
9133                 ack_am = (th_ack - rsm->r_start);
9134                 if (ack_am >= rsm->r_rtr_bytes) {
9135                         rack->r_ctl.rc_holes_rxt -= ack_am;
9136                         rsm->r_rtr_bytes -= ack_am;
9137                 }
9138         }
9139         /*
9140          * Update where the piece starts and record
9141          * the time of send of highest cumack sent.
9142          */
9143         rack->r_ctl.rc_gp_cumack_ts = rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
9144         rack_log_map_chg(tp, rack, NULL, rsm, NULL, MAP_TRIM_HEAD, th_ack, __LINE__);
9145         /* Now we need to move our offset forward too */
9146         if (rsm->m && (rsm->orig_m_len != rsm->m->m_len)) {
9147                 /* Fix up the orig_m_len and possibly the mbuf offset */
9148                 rack_adjust_orig_mlen(rsm);
9149         }
9150         rsm->soff += (th_ack - rsm->r_start);
9151         rsm->r_start = th_ack;
9152         /* Now do we need to move the mbuf fwd too? */
9153         if (rsm->m) {
9154                 while (rsm->soff >= rsm->m->m_len) {
9155                         rsm->soff -= rsm->m->m_len;
9156                         rsm->m = rsm->m->m_next;
9157                         KASSERT((rsm->m != NULL),
9158                                 (" nrsm:%p hit at soff:%u null m",
9159                                  rsm, rsm->soff));
9160                 }
9161                 rsm->orig_m_len = rsm->m->m_len;
9162         }
9163         if (rack->app_limited_needs_set)
9164                 rack_need_set_test(tp, rack, rsm, tp->snd_una, __LINE__, RACK_USE_BEG);
9165 }
9166
9167 static void
9168 rack_handle_might_revert(struct tcpcb *tp, struct tcp_rack *rack)
9169 {
9170         struct rack_sendmap *rsm;
9171         int sack_pass_fnd = 0;
9172
9173         if (rack->r_might_revert) {
9174                 /*
9175                  * Ok we have reordering, have not sent anything, we
9176                  * might want to revert the congestion state if nothing
9177                  * further has SACK_PASSED on it. Lets check.
9178                  *
9179                  * We also get here when we have DSACKs come in for
9180                  * all the data that we FR'd. Note that a rxt or tlp
9181                  * timer clears this from happening.
9182                  */
9183
9184                 TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) {
9185                         if (rsm->r_flags & RACK_SACK_PASSED) {
9186                                 sack_pass_fnd = 1;
9187                                 break;
9188                         }
9189                 }
9190                 if (sack_pass_fnd == 0) {
9191                         /*
9192                          * We went into recovery
9193                          * incorrectly due to reordering!
9194                          */
9195                         int orig_cwnd;
9196
9197                         rack->r_ent_rec_ns = 0;
9198                         orig_cwnd = tp->snd_cwnd;
9199                         tp->snd_ssthresh = rack->r_ctl.rc_ssthresh_at_erec;
9200                         tp->snd_recover = tp->snd_una;
9201                         rack_log_to_prr(rack, 14, orig_cwnd, __LINE__);
9202                         EXIT_RECOVERY(tp->t_flags);
9203                 }
9204                 rack->r_might_revert = 0;
9205         }
9206 }
9207
9208 #ifdef NETFLIX_EXP_DETECTION
9209 static void
9210 rack_do_detection(struct tcpcb *tp, struct tcp_rack *rack,  uint32_t bytes_this_ack, uint32_t segsiz)
9211 {
9212         if ((rack->do_detection || tcp_force_detection) &&
9213             tcp_sack_to_ack_thresh &&
9214             tcp_sack_to_move_thresh &&
9215             ((rack->r_ctl.rc_num_maps_alloced > tcp_map_minimum) || rack->sack_attack_disable)) {
9216                 /*
9217                  * We have thresholds set to find
9218                  * possible attackers and disable sack.
9219                  * Check them.
9220                  */
9221                 uint64_t ackratio, moveratio, movetotal;
9222
9223                 /* Log detecting */
9224                 rack_log_sad(rack, 1);
9225                 ackratio = (uint64_t)(rack->r_ctl.sack_count);
9226                 ackratio *= (uint64_t)(1000);
9227                 if (rack->r_ctl.ack_count)
9228                         ackratio /= (uint64_t)(rack->r_ctl.ack_count);
9229                 else {
9230                         /* We really should not hit here */
9231                         ackratio = 1000;
9232                 }
9233                 if ((rack->sack_attack_disable == 0) &&
9234                     (ackratio > rack_highest_sack_thresh_seen))
9235                         rack_highest_sack_thresh_seen = (uint32_t)ackratio;
9236                 movetotal = rack->r_ctl.sack_moved_extra;
9237                 movetotal += rack->r_ctl.sack_noextra_move;
9238                 moveratio = rack->r_ctl.sack_moved_extra;
9239                 moveratio *= (uint64_t)1000;
9240                 if (movetotal)
9241                         moveratio /= movetotal;
9242                 else {
9243                         /* No moves, thats pretty good */
9244                         moveratio = 0;
9245                 }
9246                 if ((rack->sack_attack_disable == 0) &&
9247                     (moveratio > rack_highest_move_thresh_seen))
9248                         rack_highest_move_thresh_seen = (uint32_t)moveratio;
9249                 if (rack->sack_attack_disable == 0) {
9250                         if ((ackratio > tcp_sack_to_ack_thresh) &&
9251                             (moveratio > tcp_sack_to_move_thresh)) {
9252                                 /* Disable sack processing */
9253                                 rack->sack_attack_disable = 1;
9254                                 if (rack->r_rep_attack == 0) {
9255                                         rack->r_rep_attack = 1;
9256                                         counter_u64_add(rack_sack_attacks_detected, 1);
9257                                 }
9258                                 if (tcp_attack_on_turns_on_logging) {
9259                                         /*
9260                                          * Turn on logging, used for debugging
9261                                          * false positives.
9262                                          */
9263                                         rack->rc_tp->t_logstate = tcp_attack_on_turns_on_logging;
9264                                 }
9265                                 /* Clamp the cwnd at flight size */
9266                                 rack->r_ctl.rc_saved_cwnd = rack->rc_tp->snd_cwnd;
9267                                 rack->rc_tp->snd_cwnd = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
9268                                 rack_log_sad(rack, 2);
9269                         }
9270                 } else {
9271                         /* We are sack-disabled check for false positives */
9272                         if ((ackratio <= tcp_restoral_thresh) ||
9273                             (rack->r_ctl.rc_num_maps_alloced  < tcp_map_minimum)) {
9274                                 rack->sack_attack_disable = 0;
9275                                 rack_log_sad(rack, 3);
9276                                 /* Restart counting */
9277                                 rack->r_ctl.sack_count = 0;
9278                                 rack->r_ctl.sack_moved_extra = 0;
9279                                 rack->r_ctl.sack_noextra_move = 1;
9280                                 rack->r_ctl.ack_count = max(1,
9281                                       (bytes_this_ack / segsiz));
9282
9283                                 if (rack->r_rep_reverse == 0) {
9284                                         rack->r_rep_reverse = 1;
9285                                         counter_u64_add(rack_sack_attacks_reversed, 1);
9286                                 }
9287                                 /* Restore the cwnd */
9288                                 if (rack->r_ctl.rc_saved_cwnd > rack->rc_tp->snd_cwnd)
9289                                         rack->rc_tp->snd_cwnd = rack->r_ctl.rc_saved_cwnd;
9290                         }
9291                 }
9292         }
9293 }
9294 #endif
9295
9296 static int
9297 rack_note_dsack(struct tcp_rack *rack, tcp_seq start, tcp_seq end)
9298 {
9299
9300         uint32_t am, l_end;
9301         int was_tlp = 0;
9302
9303         if (SEQ_GT(end, start))
9304                 am = end - start;
9305         else
9306                 am = 0;
9307         if ((rack->rc_last_tlp_acked_set ) &&
9308             (SEQ_GEQ(start, rack->r_ctl.last_tlp_acked_start)) &&
9309             (SEQ_LEQ(end, rack->r_ctl.last_tlp_acked_end))) {
9310                 /*
9311                  * The DSACK is because of a TLP which we don't
9312                  * do anything with the reordering window over since
9313                  * it was not reordering that caused the DSACK but
9314                  * our previous retransmit TLP.
9315                  */
9316                 rack_log_dsack_event(rack, 7, __LINE__, start, end);
9317                 was_tlp = 1;
9318                 goto skip_dsack_round;
9319         }
9320         if (rack->rc_last_sent_tlp_seq_valid) {
9321                 l_end = rack->r_ctl.last_sent_tlp_seq + rack->r_ctl.last_sent_tlp_len;
9322                 if (SEQ_GEQ(start, rack->r_ctl.last_sent_tlp_seq) &&
9323                     (SEQ_LEQ(end, l_end))) {
9324                         /*
9325                          * This dsack is from the last sent TLP, ignore it
9326                          * for reordering purposes.
9327                          */
9328                         rack_log_dsack_event(rack, 7, __LINE__, start, end);
9329                         was_tlp = 1;
9330                         goto skip_dsack_round;
9331                 }
9332         }
9333         if (rack->rc_dsack_round_seen == 0) {
9334                 rack->rc_dsack_round_seen = 1;
9335                 rack->r_ctl.dsack_round_end = rack->rc_tp->snd_max;
9336                 rack->r_ctl.num_dsack++;
9337                 rack->r_ctl.dsack_persist = 16; /* 16 is from the standard */
9338                 rack_log_dsack_event(rack, 2, __LINE__, 0, 0);
9339         }
9340 skip_dsack_round:
9341         /*
9342          * We keep track of how many DSACK blocks we get
9343          * after a recovery incident.
9344          */
9345         rack->r_ctl.dsack_byte_cnt += am;
9346         if (!IN_FASTRECOVERY(rack->rc_tp->t_flags) &&
9347             rack->r_ctl.retran_during_recovery &&
9348             (rack->r_ctl.dsack_byte_cnt >= rack->r_ctl.retran_during_recovery)) {
9349                 /*
9350                  * False recovery most likely culprit is reordering. If
9351                  * nothing else is missing we need to revert.
9352                  */
9353                 rack->r_might_revert = 1;
9354                 rack_handle_might_revert(rack->rc_tp, rack);
9355                 rack->r_might_revert = 0;
9356                 rack->r_ctl.retran_during_recovery = 0;
9357                 rack->r_ctl.dsack_byte_cnt = 0;
9358         }
9359         return (was_tlp);
9360 }
9361
9362 static void
9363 rack_update_prr(struct tcpcb *tp, struct tcp_rack *rack, uint32_t changed, tcp_seq th_ack)
9364 {
9365         /* Deal with changed and PRR here (in recovery only) */
9366         uint32_t pipe, snd_una;
9367
9368         rack->r_ctl.rc_prr_delivered += changed;
9369
9370         if (sbavail(&rack->rc_inp->inp_socket->so_snd) <= (tp->snd_max - tp->snd_una)) {
9371                 /*
9372                  * It is all outstanding, we are application limited
9373                  * and thus we don't need more room to send anything.
9374                  * Note we use tp->snd_una here and not th_ack because
9375                  * the data as yet not been cut from the sb.
9376                  */
9377                 rack->r_ctl.rc_prr_sndcnt = 0;
9378                 return;
9379         }
9380         /* Compute prr_sndcnt */
9381         if (SEQ_GT(tp->snd_una, th_ack)) {
9382                 snd_una = tp->snd_una;
9383         } else {
9384                 snd_una = th_ack;
9385         }
9386         pipe = ((tp->snd_max - snd_una) - rack->r_ctl.rc_sacked) + rack->r_ctl.rc_holes_rxt;
9387         if (pipe > tp->snd_ssthresh) {
9388                 long sndcnt;
9389
9390                 sndcnt = rack->r_ctl.rc_prr_delivered * tp->snd_ssthresh;
9391                 if (rack->r_ctl.rc_prr_recovery_fs > 0)
9392                         sndcnt /= (long)rack->r_ctl.rc_prr_recovery_fs;
9393                 else {
9394                         rack->r_ctl.rc_prr_sndcnt = 0;
9395                         rack_log_to_prr(rack, 9, 0, __LINE__);
9396                         sndcnt = 0;
9397                 }
9398                 sndcnt++;
9399                 if (sndcnt > (long)rack->r_ctl.rc_prr_out)
9400                         sndcnt -= rack->r_ctl.rc_prr_out;
9401                 else
9402                         sndcnt = 0;
9403                 rack->r_ctl.rc_prr_sndcnt = sndcnt;
9404                 rack_log_to_prr(rack, 10, 0, __LINE__);
9405         } else {
9406                 uint32_t limit;
9407
9408                 if (rack->r_ctl.rc_prr_delivered > rack->r_ctl.rc_prr_out)
9409                         limit = (rack->r_ctl.rc_prr_delivered - rack->r_ctl.rc_prr_out);
9410                 else
9411                         limit = 0;
9412                 if (changed > limit)
9413                         limit = changed;
9414                 limit += ctf_fixed_maxseg(tp);
9415                 if (tp->snd_ssthresh > pipe) {
9416                         rack->r_ctl.rc_prr_sndcnt = min((tp->snd_ssthresh - pipe), limit);
9417                         rack_log_to_prr(rack, 11, 0, __LINE__);
9418                 } else {
9419                         rack->r_ctl.rc_prr_sndcnt = min(0, limit);
9420                         rack_log_to_prr(rack, 12, 0, __LINE__);
9421                 }
9422         }
9423 }
9424
9425 static void
9426 rack_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, int entered_recovery, int dup_ack_struck)
9427 {
9428         uint32_t changed;
9429         struct tcp_rack *rack;
9430         struct rack_sendmap *rsm;
9431         struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1];
9432         register uint32_t th_ack;
9433         int32_t i, j, k, num_sack_blks = 0;
9434         uint32_t cts, acked, ack_point;
9435         int loop_start = 0, moved_two = 0;
9436         uint32_t tsused;
9437
9438
9439         INP_WLOCK_ASSERT(tp->t_inpcb);
9440         if (tcp_get_flags(th) & TH_RST) {
9441                 /* We don't log resets */
9442                 return;
9443         }
9444         rack = (struct tcp_rack *)tp->t_fb_ptr;
9445         cts = tcp_get_usecs(NULL);
9446         rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
9447         changed = 0;
9448         th_ack = th->th_ack;
9449         if (rack->sack_attack_disable == 0)
9450                 rack_do_decay(rack);
9451         if (BYTES_THIS_ACK(tp, th) >= ctf_fixed_maxseg(rack->rc_tp)) {
9452                 /*
9453                  * You only get credit for
9454                  * MSS and greater (and you get extra
9455                  * credit for larger cum-ack moves).
9456                  */
9457                 int ac;
9458
9459                 ac = BYTES_THIS_ACK(tp, th) / ctf_fixed_maxseg(rack->rc_tp);
9460                 rack->r_ctl.ack_count += ac;
9461                 counter_u64_add(rack_ack_total, ac);
9462         }
9463         if (rack->r_ctl.ack_count > 0xfff00000) {
9464                 /*
9465                  * reduce the number to keep us under
9466                  * a uint32_t.
9467                  */
9468                 rack->r_ctl.ack_count /= 2;
9469                 rack->r_ctl.sack_count /= 2;
9470         }
9471         if (SEQ_GT(th_ack, tp->snd_una)) {
9472                 rack_log_progress_event(rack, tp, ticks, PROGRESS_UPDATE, __LINE__);
9473                 tp->t_acktime = ticks;
9474         }
9475         if (rsm && SEQ_GT(th_ack, rsm->r_start))
9476                 changed = th_ack - rsm->r_start;
9477         if (changed) {
9478                 rack_process_to_cumack(tp, rack, th_ack, cts, to);
9479         }
9480         if ((to->to_flags & TOF_SACK) == 0) {
9481                 /* We are done nothing left and no sack. */
9482                 rack_handle_might_revert(tp, rack);
9483                 /*
9484                  * For cases where we struck a dup-ack
9485                  * with no SACK, add to the changes so
9486                  * PRR will work right.
9487                  */
9488                 if (dup_ack_struck && (changed == 0)) {
9489                         changed += ctf_fixed_maxseg(rack->rc_tp);
9490                 }
9491                 goto out;
9492         }
9493         /* Sack block processing */
9494         if (SEQ_GT(th_ack, tp->snd_una))
9495                 ack_point = th_ack;
9496         else
9497                 ack_point = tp->snd_una;
9498         for (i = 0; i < to->to_nsacks; i++) {
9499                 bcopy((to->to_sacks + i * TCPOLEN_SACK),
9500                       &sack, sizeof(sack));
9501                 sack.start = ntohl(sack.start);
9502                 sack.end = ntohl(sack.end);
9503                 if (SEQ_GT(sack.end, sack.start) &&
9504                     SEQ_GT(sack.start, ack_point) &&
9505                     SEQ_LT(sack.start, tp->snd_max) &&
9506                     SEQ_GT(sack.end, ack_point) &&
9507                     SEQ_LEQ(sack.end, tp->snd_max)) {
9508                         sack_blocks[num_sack_blks] = sack;
9509                         num_sack_blks++;
9510                 } else if (SEQ_LEQ(sack.start, th_ack) &&
9511                            SEQ_LEQ(sack.end, th_ack)) {
9512                         int was_tlp;
9513
9514                         was_tlp = rack_note_dsack(rack, sack.start, sack.end);
9515                         /*
9516                          * Its a D-SACK block.
9517                          */
9518                         tcp_record_dsack(tp, sack.start, sack.end, was_tlp);
9519                 }
9520         }
9521         if (rack->rc_dsack_round_seen) {
9522                 /* Is the dsack roound over? */
9523                 if (SEQ_GEQ(th_ack, rack->r_ctl.dsack_round_end)) {
9524                         /* Yes it is */
9525                         rack->rc_dsack_round_seen = 0;
9526                         rack_log_dsack_event(rack, 3, __LINE__, 0, 0);
9527                 }
9528         }
9529         /*
9530          * Sort the SACK blocks so we can update the rack scoreboard with
9531          * just one pass.
9532          */
9533         num_sack_blks = sack_filter_blks(&rack->r_ctl.rack_sf, sack_blocks,
9534                                          num_sack_blks, th->th_ack);
9535         ctf_log_sack_filter(rack->rc_tp, num_sack_blks, sack_blocks);
9536         if (num_sack_blks == 0) {
9537                 /* Nothing to sack (DSACKs?) */
9538                 goto out_with_totals;
9539         }
9540         if (num_sack_blks < 2) {
9541                 /* Only one, we don't need to sort */
9542                 goto do_sack_work;
9543         }
9544         /* Sort the sacks */
9545         for (i = 0; i < num_sack_blks; i++) {
9546                 for (j = i + 1; j < num_sack_blks; j++) {
9547                         if (SEQ_GT(sack_blocks[i].end, sack_blocks[j].end)) {
9548                                 sack = sack_blocks[i];
9549                                 sack_blocks[i] = sack_blocks[j];
9550                                 sack_blocks[j] = sack;
9551                         }
9552                 }
9553         }
9554         /*
9555          * Now are any of the sack block ends the same (yes some
9556          * implementations send these)?
9557          */
9558 again:
9559         if (num_sack_blks == 0)
9560                 goto out_with_totals;
9561         if (num_sack_blks > 1) {
9562                 for (i = 0; i < num_sack_blks; i++) {
9563                         for (j = i + 1; j < num_sack_blks; j++) {
9564                                 if (sack_blocks[i].end == sack_blocks[j].end) {
9565                                         /*
9566                                          * Ok these two have the same end we
9567                                          * want the smallest end and then
9568                                          * throw away the larger and start
9569                                          * again.
9570                                          */
9571                                         if (SEQ_LT(sack_blocks[j].start, sack_blocks[i].start)) {
9572                                                 /*
9573                                                  * The second block covers
9574                                                  * more area use that
9575                                                  */
9576                                                 sack_blocks[i].start = sack_blocks[j].start;
9577                                         }
9578                                         /*
9579                                          * Now collapse out the dup-sack and
9580                                          * lower the count
9581                                          */
9582                                         for (k = (j + 1); k < num_sack_blks; k++) {
9583                                                 sack_blocks[j].start = sack_blocks[k].start;
9584                                                 sack_blocks[j].end = sack_blocks[k].end;
9585                                                 j++;
9586                                         }
9587                                         num_sack_blks--;
9588                                         goto again;
9589                                 }
9590                         }
9591                 }
9592         }
9593 do_sack_work:
9594         /*
9595          * First lets look to see if
9596          * we have retransmitted and
9597          * can use the transmit next?
9598          */
9599         rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
9600         if (rsm &&
9601             SEQ_GT(sack_blocks[0].end, rsm->r_start) &&
9602             SEQ_LT(sack_blocks[0].start, rsm->r_end)) {
9603                 /*
9604                  * We probably did the FR and the next
9605                  * SACK in continues as we would expect.
9606                  */
9607                 acked = rack_proc_sack_blk(tp, rack, &sack_blocks[0], to, &rsm, cts, &moved_two);
9608                 if (acked) {
9609                         rack->r_wanted_output = 1;
9610                         changed += acked;
9611                 }
9612                 if (num_sack_blks == 1) {
9613                         /*
9614                          * This is what we would expect from
9615                          * a normal implementation to happen
9616                          * after we have retransmitted the FR,
9617                          * i.e the sack-filter pushes down
9618                          * to 1 block and the next to be retransmitted
9619                          * is the sequence in the sack block (has more
9620                          * are acked). Count this as ACK'd data to boost
9621                          * up the chances of recovering any false positives.
9622                          */
9623                         rack->r_ctl.ack_count += (acked / ctf_fixed_maxseg(rack->rc_tp));
9624                         counter_u64_add(rack_ack_total, (acked / ctf_fixed_maxseg(rack->rc_tp)));
9625                         counter_u64_add(rack_express_sack, 1);
9626                         if (rack->r_ctl.ack_count > 0xfff00000) {
9627                                 /*
9628                                  * reduce the number to keep us under
9629                                  * a uint32_t.
9630                                  */
9631                                 rack->r_ctl.ack_count /= 2;
9632                                 rack->r_ctl.sack_count /= 2;
9633                         }
9634                         goto out_with_totals;
9635                 } else {
9636                         /*
9637                          * Start the loop through the
9638                          * rest of blocks, past the first block.
9639                          */
9640                         moved_two = 0;
9641                         loop_start = 1;
9642                 }
9643         }
9644         /* Its a sack of some sort */
9645         rack->r_ctl.sack_count++;
9646         if (rack->r_ctl.sack_count > 0xfff00000) {
9647                 /*
9648                  * reduce the number to keep us under
9649                  * a uint32_t.
9650                  */
9651                 rack->r_ctl.ack_count /= 2;
9652                 rack->r_ctl.sack_count /= 2;
9653         }
9654         counter_u64_add(rack_sack_total, 1);
9655         if (rack->sack_attack_disable) {
9656                 /* An attacker disablement is in place */
9657                 if (num_sack_blks > 1) {
9658                         rack->r_ctl.sack_count += (num_sack_blks - 1);
9659                         rack->r_ctl.sack_moved_extra++;
9660                         counter_u64_add(rack_move_some, 1);
9661                         if (rack->r_ctl.sack_moved_extra > 0xfff00000) {
9662                                 rack->r_ctl.sack_moved_extra /= 2;
9663                                 rack->r_ctl.sack_noextra_move /= 2;
9664                         }
9665                 }
9666                 goto out;
9667         }
9668         rsm = rack->r_ctl.rc_sacklast;
9669         for (i = loop_start; i < num_sack_blks; i++) {
9670                 acked = rack_proc_sack_blk(tp, rack, &sack_blocks[i], to, &rsm, cts, &moved_two);
9671                 if (acked) {
9672                         rack->r_wanted_output = 1;
9673                         changed += acked;
9674                 }
9675                 if (moved_two) {
9676                         /*
9677                          * If we did not get a SACK for at least a MSS and
9678                          * had to move at all, or if we moved more than our
9679                          * threshold, it counts against the "extra" move.
9680                          */
9681                         rack->r_ctl.sack_moved_extra += moved_two;
9682                         counter_u64_add(rack_move_some, 1);
9683                 } else {
9684                         /*
9685                          * else we did not have to move
9686                          * any more than we would expect.
9687                          */
9688                         rack->r_ctl.sack_noextra_move++;
9689                         counter_u64_add(rack_move_none, 1);
9690                 }
9691                 if (moved_two && (acked < ctf_fixed_maxseg(rack->rc_tp))) {
9692                         /*
9693                          * If the SACK was not a full MSS then
9694                          * we add to sack_count the number of
9695                          * MSS's (or possibly more than
9696                          * a MSS if its a TSO send) we had to skip by.
9697                          */
9698                         rack->r_ctl.sack_count += moved_two;
9699                         counter_u64_add(rack_sack_total, moved_two);
9700                 }
9701                 /*
9702                  * Now we need to setup for the next
9703                  * round. First we make sure we won't
9704                  * exceed the size of our uint32_t on
9705                  * the various counts, and then clear out
9706                  * moved_two.
9707                  */
9708                 if ((rack->r_ctl.sack_moved_extra > 0xfff00000) ||
9709                     (rack->r_ctl.sack_noextra_move > 0xfff00000)) {
9710                         rack->r_ctl.sack_moved_extra /= 2;
9711                         rack->r_ctl.sack_noextra_move /= 2;
9712                 }
9713                 if (rack->r_ctl.sack_count > 0xfff00000) {
9714                         rack->r_ctl.ack_count /= 2;
9715                         rack->r_ctl.sack_count /= 2;
9716                 }
9717                 moved_two = 0;
9718         }
9719 out_with_totals:
9720         if (num_sack_blks > 1) {
9721                 /*
9722                  * You get an extra stroke if
9723                  * you have more than one sack-blk, this
9724                  * could be where we are skipping forward
9725                  * and the sack-filter is still working, or
9726                  * it could be an attacker constantly
9727                  * moving us.
9728                  */
9729                 rack->r_ctl.sack_moved_extra++;
9730                 counter_u64_add(rack_move_some, 1);
9731         }
9732 out:
9733 #ifdef NETFLIX_EXP_DETECTION
9734         rack_do_detection(tp, rack, BYTES_THIS_ACK(tp, th), ctf_fixed_maxseg(rack->rc_tp));
9735 #endif
9736         if (changed) {
9737                 /* Something changed cancel the rack timer */
9738                 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
9739         }
9740         tsused = tcp_get_usecs(NULL);
9741         rsm = tcp_rack_output(tp, rack, tsused);
9742         if ((!IN_FASTRECOVERY(tp->t_flags)) &&
9743             rsm &&
9744             ((rsm->r_flags & RACK_MUST_RXT) == 0)) {
9745                 /* Enter recovery */
9746                 entered_recovery = 1;
9747                 rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__);
9748                 /*
9749                  * When we enter recovery we need to assure we send
9750                  * one packet.
9751                  */
9752                 if (rack->rack_no_prr == 0) {
9753                         rack->r_ctl.rc_prr_sndcnt = ctf_fixed_maxseg(tp);
9754                         rack_log_to_prr(rack, 8, 0, __LINE__);
9755                 }
9756                 rack->r_timer_override = 1;
9757                 rack->r_early = 0;
9758                 rack->r_ctl.rc_agg_early = 0;
9759         } else if (IN_FASTRECOVERY(tp->t_flags) &&
9760                    rsm &&
9761                    (rack->r_rr_config == 3)) {
9762                 /*
9763                  * Assure we can output and we get no
9764                  * remembered pace time except the retransmit.
9765                  */
9766                 rack->r_timer_override = 1;
9767                 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
9768                 rack->r_ctl.rc_resend = rsm;
9769         }
9770         if (IN_FASTRECOVERY(tp->t_flags) &&
9771             (rack->rack_no_prr == 0) &&
9772             (entered_recovery == 0)) {
9773                 rack_update_prr(tp, rack, changed, th_ack);
9774                 if ((rsm && (rack->r_ctl.rc_prr_sndcnt >= ctf_fixed_maxseg(tp)) &&
9775                      ((tcp_in_hpts(rack->rc_inp) == 0) &&
9776                       ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0)))) {
9777                         /*
9778                          * If you are pacing output you don't want
9779                          * to override.
9780                          */
9781                         rack->r_early = 0;
9782                         rack->r_ctl.rc_agg_early = 0;
9783                         rack->r_timer_override = 1;
9784                 }
9785         }
9786 }
9787
9788 static void
9789 rack_strike_dupack(struct tcp_rack *rack)
9790 {
9791         struct rack_sendmap *rsm;
9792
9793         rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
9794         while (rsm && (rsm->r_dupack >= DUP_ACK_THRESHOLD)) {
9795                 rsm = TAILQ_NEXT(rsm, r_tnext);
9796                 if (rsm->r_flags & RACK_MUST_RXT) {
9797                         /* Sendmap entries that are marked to
9798                          * be retransmitted do not need dupack's
9799                          * struck. We get these marks for a number
9800                          * of reasons (rxt timeout with no sack,
9801                          * mtu change, or rwnd collapses). When
9802                          * these events occur, we know we must retransmit
9803                          * them and mark the sendmap entries. Dupack counting
9804                          * is not needed since we are already set to retransmit
9805                          * it as soon as we can.
9806                          */
9807                         continue;
9808                 }
9809         }
9810         if (rsm && (rsm->r_dupack < 0xff)) {
9811                 rsm->r_dupack++;
9812                 if (rsm->r_dupack >= DUP_ACK_THRESHOLD) {
9813                         struct timeval tv;
9814                         uint32_t cts;
9815                         /*
9816                          * Here we see if we need to retransmit. For
9817                          * a SACK type connection if enough time has passed
9818                          * we will get a return of the rsm. For a non-sack
9819                          * connection we will get the rsm returned if the
9820                          * dupack value is 3 or more.
9821                          */
9822                         cts = tcp_get_usecs(&tv);
9823                         rack->r_ctl.rc_resend = tcp_rack_output(rack->rc_tp, rack, cts);
9824                         if (rack->r_ctl.rc_resend != NULL) {
9825                                 if (!IN_FASTRECOVERY(rack->rc_tp->t_flags)) {
9826                                         rack_cong_signal(rack->rc_tp, CC_NDUPACK,
9827                                                          rack->rc_tp->snd_una, __LINE__);
9828                                 }
9829                                 rack->r_wanted_output = 1;
9830                                 rack->r_timer_override = 1;
9831                                 rack_log_retran_reason(rack, rsm, __LINE__, 1, 3);
9832                         }
9833                 } else {
9834                         rack_log_retran_reason(rack, rsm, __LINE__, 0, 3);
9835                 }
9836         }
9837 }
9838
9839 static void
9840 rack_check_bottom_drag(struct tcpcb *tp,
9841                        struct tcp_rack *rack,
9842                        struct socket *so, int32_t acked)
9843 {
9844         uint32_t segsiz, minseg;
9845
9846         segsiz = ctf_fixed_maxseg(tp);
9847         minseg = segsiz;
9848
9849         if (tp->snd_max == tp->snd_una) {
9850                 /*
9851                  * We are doing dynamic pacing and we are way
9852                  * under. Basically everything got acked while
9853                  * we were still waiting on the pacer to expire.
9854                  *
9855                  * This means we need to boost the b/w in
9856                  * addition to any earlier boosting of
9857                  * the multiplier.
9858                  */
9859                 rack->rc_dragged_bottom = 1;
9860                 rack_validate_multipliers_at_or_above100(rack);
9861                 /*
9862                  * Lets use the segment bytes acked plus
9863                  * the lowest RTT seen as the basis to
9864                  * form a b/w estimate. This will be off
9865                  * due to the fact that the true estimate
9866                  * should be around 1/2 the time of the RTT
9867                  * but we can settle for that.
9868                  */
9869                 if ((rack->r_ctl.rack_rs.rs_flags & RACK_RTT_VALID) &&
9870                     acked) {
9871                         uint64_t bw, calc_bw, rtt;
9872
9873                         rtt = rack->r_ctl.rack_rs.rs_us_rtt;
9874                         if (rtt == 0) {
9875                                 /* no us sample is there a ms one? */
9876                                 if (rack->r_ctl.rack_rs.rs_rtt_lowest) {
9877                                         rtt = rack->r_ctl.rack_rs.rs_rtt_lowest;
9878                                 } else {
9879                                         goto no_measurement;
9880                                 }
9881                         }
9882                         bw = acked;
9883                         calc_bw = bw * 1000000;
9884                         calc_bw /= rtt;
9885                         if (rack->r_ctl.last_max_bw &&
9886                             (rack->r_ctl.last_max_bw < calc_bw)) {
9887                                 /*
9888                                  * If we have a last calculated max bw
9889                                  * enforce it.
9890                                  */
9891                                 calc_bw = rack->r_ctl.last_max_bw;
9892                         }
9893                         /* now plop it in */
9894                         if (rack->rc_gp_filled == 0) {
9895                                 if (calc_bw > ONE_POINT_TWO_MEG) {
9896                                         /*
9897                                          * If we have no measurement
9898                                          * don't let us set in more than
9899                                          * 1.2Mbps. If we are still too
9900                                          * low after pacing with this we
9901                                          * will hopefully have a max b/w
9902                                          * available to sanity check things.
9903                                          */
9904                                         calc_bw = ONE_POINT_TWO_MEG;
9905                                 }
9906                                 rack->r_ctl.rc_rtt_diff = 0;
9907                                 rack->r_ctl.gp_bw = calc_bw;
9908                                 rack->rc_gp_filled = 1;
9909                                 if (rack->r_ctl.num_measurements < RACK_REQ_AVG)
9910                                         rack->r_ctl.num_measurements = RACK_REQ_AVG;
9911                                 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
9912                         } else if (calc_bw > rack->r_ctl.gp_bw) {
9913                                 rack->r_ctl.rc_rtt_diff = 0;
9914                                 if (rack->r_ctl.num_measurements < RACK_REQ_AVG)
9915                                         rack->r_ctl.num_measurements = RACK_REQ_AVG;
9916                                 rack->r_ctl.gp_bw = calc_bw;
9917                                 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
9918                         } else
9919                                 rack_increase_bw_mul(rack, -1, 0, 0, 1);
9920                         if ((rack->gp_ready == 0) &&
9921                             (rack->r_ctl.num_measurements >= rack->r_ctl.req_measurements)) {
9922                                 /* We have enough measurements now */
9923                                 rack->gp_ready = 1;
9924                                 rack_set_cc_pacing(rack);
9925                                 if (rack->defer_options)
9926                                         rack_apply_deferred_options(rack);
9927                         }
9928                         /*
9929                          * For acks over 1mss we do a extra boost to simulate
9930                          * where we would get 2 acks (we want 110 for the mul).
9931                          */
9932                         if (acked > segsiz)
9933                                 rack_increase_bw_mul(rack, -1, 0, 0, 1);
9934                 } else {
9935                         /*
9936                          * zero rtt possibly?, settle for just an old increase.
9937                          */
9938 no_measurement:
9939                         rack_increase_bw_mul(rack, -1, 0, 0, 1);
9940                 }
9941         } else if ((IN_FASTRECOVERY(tp->t_flags) == 0) &&
9942                    (sbavail(&so->so_snd) > max((segsiz * (4 + rack_req_segs)),
9943                                                minseg)) &&
9944                    (rack->r_ctl.cwnd_to_use > max((segsiz * (rack_req_segs + 2)), minseg)) &&
9945                    (tp->snd_wnd > max((segsiz * (rack_req_segs + 2)), minseg)) &&
9946                    (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) <=
9947                     (segsiz * rack_req_segs))) {
9948                 /*
9949                  * We are doing dynamic GP pacing and
9950                  * we have everything except 1MSS or less
9951                  * bytes left out. We are still pacing away.
9952                  * And there is data that could be sent, This
9953                  * means we are inserting delayed ack time in
9954                  * our measurements because we are pacing too slow.
9955                  */
9956                 rack_validate_multipliers_at_or_above100(rack);
9957                 rack->rc_dragged_bottom = 1;
9958                 rack_increase_bw_mul(rack, -1, 0, 0, 1);
9959         }
9960 }
9961
9962
9963
9964 static void
9965 rack_gain_for_fastoutput(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_t acked_amount)
9966 {
9967         /*
9968          * The fast output path is enabled and we
9969          * have moved the cumack forward. Lets see if
9970          * we can expand forward the fast path length by
9971          * that amount. What we would ideally like to
9972          * do is increase the number of bytes in the
9973          * fast path block (left_to_send) by the
9974          * acked amount. However we have to gate that
9975          * by two factors:
9976          * 1) The amount outstanding and the rwnd of the peer
9977          *    (i.e. we don't want to exceed the rwnd of the peer).
9978          *    <and>
9979          * 2) The amount of data left in the socket buffer (i.e.
9980          *    we can't send beyond what is in the buffer).
9981          *
9982          * Note that this does not take into account any increase
9983          * in the cwnd. We will only extend the fast path by
9984          * what was acked.
9985          */
9986         uint32_t new_total, gating_val;
9987
9988         new_total = acked_amount + rack->r_ctl.fsb.left_to_send;
9989         gating_val = min((sbavail(&so->so_snd) - (tp->snd_max - tp->snd_una)),
9990                          (tp->snd_wnd - (tp->snd_max - tp->snd_una)));
9991         if (new_total <= gating_val) {
9992                 /* We can increase left_to_send by the acked amount */
9993                 counter_u64_add(rack_extended_rfo, 1);
9994                 rack->r_ctl.fsb.left_to_send = new_total;
9995                 KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(&rack->rc_inp->inp_socket->so_snd) - (tp->snd_max - tp->snd_una))),
9996                         ("rack:%p left_to_send:%u sbavail:%u out:%u",
9997                          rack, rack->r_ctl.fsb.left_to_send,
9998                          sbavail(&rack->rc_inp->inp_socket->so_snd),
9999                          (tp->snd_max - tp->snd_una)));
10000
10001         }
10002 }
10003
10004 static void
10005 rack_adjust_sendmap(struct tcp_rack *rack, struct sockbuf *sb, tcp_seq snd_una)
10006 {
10007         /*
10008          * Here any sendmap entry that points to the
10009          * beginning mbuf must be adjusted to the correct
10010          * offset. This must be called with:
10011          * 1) The socket buffer locked
10012          * 2) snd_una adjusted to its new postion.
10013          *
10014          * Note that (2) implies rack_ack_received has also
10015          * been called.
10016          *
10017          * We grab the first mbuf in the socket buffer and
10018          * then go through the front of the sendmap, recalculating
10019          * the stored offset for any sendmap entry that has
10020          * that mbuf. We must use the sb functions to do this
10021          * since its possible an add was done has well as
10022          * the subtraction we may have just completed. This should
10023          * not be a penalty though, since we just referenced the sb
10024          * to go in and trim off the mbufs that we freed (of course
10025          * there will be a penalty for the sendmap references though).
10026          */
10027         struct mbuf *m;
10028         struct rack_sendmap *rsm;
10029
10030         SOCKBUF_LOCK_ASSERT(sb);
10031         m = sb->sb_mb;
10032         rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
10033         if ((rsm == NULL) || (m == NULL)) {
10034                 /* Nothing outstanding */
10035                 return;
10036         }
10037         while (rsm->m && (rsm->m == m)) {
10038                 /* one to adjust */
10039 #ifdef INVARIANTS
10040                 struct mbuf *tm;
10041                 uint32_t soff;
10042
10043                 tm = sbsndmbuf(sb, (rsm->r_start - snd_una), &soff);
10044                 if (rsm->orig_m_len != m->m_len) {
10045                         rack_adjust_orig_mlen(rsm);
10046                 }
10047                 if (rsm->soff != soff) {
10048                         /*
10049                          * This is not a fatal error, we anticipate it
10050                          * might happen (the else code), so we count it here
10051                          * so that under invariant we can see that it really
10052                          * does happen.
10053                          */
10054                         counter_u64_add(rack_adjust_map_bw, 1);
10055                 }
10056                 rsm->m = tm;
10057                 rsm->soff = soff;
10058                 if (tm)
10059                         rsm->orig_m_len = rsm->m->m_len;
10060                 else
10061                         rsm->orig_m_len = 0;
10062 #else
10063                 rsm->m = sbsndmbuf(sb, (rsm->r_start - snd_una), &rsm->soff);
10064                 if (rsm->m)
10065                         rsm->orig_m_len = rsm->m->m_len;
10066                 else
10067                         rsm->orig_m_len = 0;
10068 #endif
10069                 rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree,
10070                               rsm);
10071                 if (rsm == NULL)
10072                         break;
10073         }
10074 }
10075
10076 /*
10077  * Return value of 1, we do not need to call rack_process_data().
10078  * return value of 0, rack_process_data can be called.
10079  * For ret_val if its 0 the TCP is locked, if its non-zero
10080  * its unlocked and probably unsafe to touch the TCB.
10081  */
10082 static int
10083 rack_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so,
10084     struct tcpcb *tp, struct tcpopt *to,
10085     uint32_t tiwin, int32_t tlen,
10086     int32_t * ofia, int32_t thflags, int32_t *ret_val)
10087 {
10088         int32_t ourfinisacked = 0;
10089         int32_t nsegs, acked_amount;
10090         int32_t acked;
10091         struct mbuf *mfree;
10092         struct tcp_rack *rack;
10093         int32_t under_pacing = 0;
10094         int32_t recovery = 0;
10095
10096         rack = (struct tcp_rack *)tp->t_fb_ptr;
10097         if (SEQ_GT(th->th_ack, tp->snd_max)) {
10098                 __ctf_do_dropafterack(m, tp, th, thflags, tlen, ret_val,
10099                                       &rack->r_ctl.challenge_ack_ts,
10100                                       &rack->r_ctl.challenge_ack_cnt);
10101                 rack->r_wanted_output = 1;
10102                 return (1);
10103         }
10104         if (rack->gp_ready &&
10105             (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
10106                 under_pacing = 1;
10107         }
10108         if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) {
10109                 int in_rec, dup_ack_struck = 0;
10110
10111                 in_rec = IN_FASTRECOVERY(tp->t_flags);
10112                 if (rack->rc_in_persist) {
10113                         tp->t_rxtshift = 0;
10114                         RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
10115                                       rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
10116                 }
10117                 if ((th->th_ack == tp->snd_una) &&
10118                     (tiwin == tp->snd_wnd) &&
10119                     ((to->to_flags & TOF_SACK) == 0)) {
10120                         rack_strike_dupack(rack);
10121                         dup_ack_struck = 1;
10122                 }
10123                 rack_log_ack(tp, to, th, ((in_rec == 0) && IN_FASTRECOVERY(tp->t_flags)), dup_ack_struck);
10124         }
10125         if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
10126                 /*
10127                  * Old ack, behind (or duplicate to) the last one rcv'd
10128                  * Note: We mark reordering is occuring if its
10129                  * less than and we have not closed our window.
10130                  */
10131                 if (SEQ_LT(th->th_ack, tp->snd_una) && (sbspace(&so->so_rcv) > ctf_fixed_maxseg(tp))) {
10132                         rack->r_ctl.rc_reorder_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time);
10133                 }
10134                 return (0);
10135         }
10136         /*
10137          * If we reach this point, ACK is not a duplicate, i.e., it ACKs
10138          * something we sent.
10139          */
10140         if (tp->t_flags & TF_NEEDSYN) {
10141                 /*
10142                  * T/TCP: Connection was half-synchronized, and our SYN has
10143                  * been ACK'd (so connection is now fully synchronized).  Go
10144                  * to non-starred state, increment snd_una for ACK of SYN,
10145                  * and check if we can do window scaling.
10146                  */
10147                 tp->t_flags &= ~TF_NEEDSYN;
10148                 tp->snd_una++;
10149                 /* Do window scaling? */
10150                 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
10151                     (TF_RCVD_SCALE | TF_REQ_SCALE)) {
10152                         tp->rcv_scale = tp->request_r_scale;
10153                         /* Send window already scaled. */
10154                 }
10155         }
10156         nsegs = max(1, m->m_pkthdr.lro_nsegs);
10157         INP_WLOCK_ASSERT(tp->t_inpcb);
10158
10159         acked = BYTES_THIS_ACK(tp, th);
10160         if (acked) {
10161                 /*
10162                  * Any time we move the cum-ack forward clear
10163                  * keep-alive tied probe-not-answered. The
10164                  * persists clears its own on entry.
10165                  */
10166                 rack->probe_not_answered = 0;
10167         }
10168         KMOD_TCPSTAT_ADD(tcps_rcvackpack, nsegs);
10169         KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
10170         /*
10171          * If we just performed our first retransmit, and the ACK arrives
10172          * within our recovery window, then it was a mistake to do the
10173          * retransmit in the first place.  Recover our original cwnd and
10174          * ssthresh, and proceed to transmit where we left off.
10175          */
10176         if ((tp->t_flags & TF_PREVVALID) &&
10177             ((tp->t_flags & TF_RCVD_TSTMP) == 0)) {
10178                 tp->t_flags &= ~TF_PREVVALID;
10179                 if (tp->t_rxtshift == 1 &&
10180                     (int)(ticks - tp->t_badrxtwin) < 0)
10181                         rack_cong_signal(tp, CC_RTO_ERR, th->th_ack, __LINE__);
10182         }
10183         if (acked) {
10184                 /* assure we are not backed off */
10185                 tp->t_rxtshift = 0;
10186                 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
10187                               rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
10188                 rack->rc_tlp_in_progress = 0;
10189                 rack->r_ctl.rc_tlp_cnt_out = 0;
10190                 /*
10191                  * If it is the RXT timer we want to
10192                  * stop it, so we can restart a TLP.
10193                  */
10194                 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)
10195                         rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
10196 #ifdef NETFLIX_HTTP_LOGGING
10197                 tcp_http_check_for_comp(rack->rc_tp, th->th_ack);
10198 #endif
10199         }
10200         /*
10201          * If we have a timestamp reply, update smoothed round trip time. If
10202          * no timestamp is present but transmit timer is running and timed
10203          * sequence number was acked, update smoothed round trip time. Since
10204          * we now have an rtt measurement, cancel the timer backoff (cf.,
10205          * Phil Karn's retransmit alg.). Recompute the initial retransmit
10206          * timer.
10207          *
10208          * Some boxes send broken timestamp replies during the SYN+ACK
10209          * phase, ignore timestamps of 0 or we could calculate a huge RTT
10210          * and blow up the retransmit timer.
10211          */
10212         /*
10213          * If all outstanding data is acked, stop retransmit timer and
10214          * remember to restart (more output or persist). If there is more
10215          * data to be acked, restart retransmit timer, using current
10216          * (possibly backed-off) value.
10217          */
10218         if (acked == 0) {
10219                 if (ofia)
10220                         *ofia = ourfinisacked;
10221                 return (0);
10222         }
10223         if (IN_RECOVERY(tp->t_flags)) {
10224                 if (SEQ_LT(th->th_ack, tp->snd_recover) &&
10225                     (SEQ_LT(th->th_ack, tp->snd_max))) {
10226                         tcp_rack_partialack(tp);
10227                 } else {
10228                         rack_post_recovery(tp, th->th_ack);
10229                         recovery = 1;
10230                 }
10231         }
10232         /*
10233          * Let the congestion control algorithm update congestion control
10234          * related information. This typically means increasing the
10235          * congestion window.
10236          */
10237         rack_ack_received(tp, rack, th->th_ack, nsegs, CC_ACK, recovery);
10238         SOCKBUF_LOCK(&so->so_snd);
10239         acked_amount = min(acked, (int)sbavail(&so->so_snd));
10240         tp->snd_wnd -= acked_amount;
10241         mfree = sbcut_locked(&so->so_snd, acked_amount);
10242         if ((sbused(&so->so_snd) == 0) &&
10243             (acked > acked_amount) &&
10244             (tp->t_state >= TCPS_FIN_WAIT_1) &&
10245             (tp->t_flags & TF_SENTFIN)) {
10246                 /*
10247                  * We must be sure our fin
10248                  * was sent and acked (we can be
10249                  * in FIN_WAIT_1 without having
10250                  * sent the fin).
10251                  */
10252                 ourfinisacked = 1;
10253         }
10254         tp->snd_una = th->th_ack;
10255         if (acked_amount && sbavail(&so->so_snd))
10256                 rack_adjust_sendmap(rack, &so->so_snd, tp->snd_una);
10257         rack_log_wakeup(tp,rack, &so->so_snd, acked, 2);
10258         /* NB: sowwakeup_locked() does an implicit unlock. */
10259         sowwakeup_locked(so);
10260         m_freem(mfree);
10261         if (SEQ_GT(tp->snd_una, tp->snd_recover))
10262                 tp->snd_recover = tp->snd_una;
10263
10264         if (SEQ_LT(tp->snd_nxt, tp->snd_una)) {
10265                 tp->snd_nxt = tp->snd_una;
10266         }
10267         if (under_pacing &&
10268             (rack->use_fixed_rate == 0) &&
10269             (rack->in_probe_rtt == 0) &&
10270             rack->rc_gp_dyn_mul &&
10271             rack->rc_always_pace) {
10272                 /* Check if we are dragging bottom */
10273                 rack_check_bottom_drag(tp, rack, so, acked);
10274         }
10275         if (tp->snd_una == tp->snd_max) {
10276                 /* Nothing left outstanding */
10277                 tp->t_flags &= ~TF_PREVVALID;
10278                 rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL);
10279                 rack->r_ctl.retran_during_recovery = 0;
10280                 rack->r_ctl.dsack_byte_cnt = 0;
10281                 if (rack->r_ctl.rc_went_idle_time == 0)
10282                         rack->r_ctl.rc_went_idle_time = 1;
10283                 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__);
10284                 if (sbavail(&tp->t_inpcb->inp_socket->so_snd) == 0)
10285                         tp->t_acktime = 0;
10286                 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
10287                 /* Set need output so persist might get set */
10288                 rack->r_wanted_output = 1;
10289                 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
10290                 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
10291                     (sbavail(&so->so_snd) == 0) &&
10292                     (tp->t_flags2 & TF2_DROP_AF_DATA)) {
10293                         /*
10294                          * The socket was gone and the
10295                          * peer sent data (now or in the past), time to
10296                          * reset him.
10297                          */
10298                         *ret_val = 1;
10299                         /* tcp_close will kill the inp pre-log the Reset */
10300                         tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
10301                         tp = tcp_close(tp);
10302                         ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, tlen);
10303                         return (1);
10304                 }
10305         }
10306         if (ofia)
10307                 *ofia = ourfinisacked;
10308         return (0);
10309 }
10310
10311 static void
10312 rack_collapsed_window(struct tcp_rack *rack)
10313 {
10314         /*
10315          * Now we must walk the
10316          * send map and divide the
10317          * ones left stranded. These
10318          * guys can't cause us to abort
10319          * the connection and are really
10320          * "unsent". However if a buggy
10321          * client actually did keep some
10322          * of the data i.e. collapsed the win
10323          * and refused to ack and then opened
10324          * the win and acked that data. We would
10325          * get into an ack war, the simplier
10326          * method then of just pretending we
10327          * did not send those segments something
10328          * won't work.
10329          */
10330         struct rack_sendmap *rsm, *nrsm, fe;
10331 #ifdef INVARIANTS
10332         struct rack_sendmap *insret;
10333 #endif
10334         tcp_seq max_seq;
10335
10336         rack_trace_point(rack, RACK_TP_COLLAPSED_WND);
10337         max_seq = rack->rc_tp->snd_una + rack->rc_tp->snd_wnd;
10338         memset(&fe, 0, sizeof(fe));
10339         fe.r_start = max_seq;
10340         /* Find the first seq past or at maxseq */
10341         rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe);
10342         if (rsm == NULL) {
10343                 /* Nothing to do strange */
10344                 rack->rc_has_collapsed = 0;
10345                 return;
10346         }
10347         /*
10348          * Now do we need to split at
10349          * the collapse point?
10350          */
10351         if (SEQ_GT(max_seq, rsm->r_start)) {
10352                 nrsm = rack_alloc_limit(rack, RACK_LIMIT_TYPE_SPLIT);
10353                 if (nrsm == NULL) {
10354                         /* We can't get a rsm, mark all? */
10355                         nrsm = rsm;
10356                         goto no_split;
10357                 }
10358                 /* Clone it */
10359                 rack_clone_rsm(rack, nrsm, rsm, max_seq);
10360 #ifndef INVARIANTS
10361                 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
10362 #else
10363                 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm);
10364                 if (insret != NULL) {
10365                         panic("Insert in rb tree of %p fails ret:%p rack:%p rsm:%p",
10366                               nrsm, insret, rack, rsm);
10367                 }
10368 #endif
10369                 rack_log_map_chg(rack->rc_tp, rack, NULL, rsm, nrsm, MAP_SPLIT, max_seq, __LINE__);
10370                 if (rsm->r_in_tmap) {
10371                         TAILQ_INSERT_AFTER(&rack->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
10372                         nrsm->r_in_tmap = 1;
10373                 }
10374                 /*
10375                  * Set in the new RSM as the
10376                  * collapsed starting point
10377                  */
10378                 rsm = nrsm;
10379         }
10380 no_split:
10381         counter_u64_add(rack_collapsed_win, 1);
10382         RB_FOREACH_FROM(nrsm, rack_rb_tree_head, rsm) {
10383                 nrsm->r_flags |= RACK_RWND_COLLAPSED;
10384         }
10385         rack->rc_has_collapsed = 1;
10386 }
10387
10388 static void
10389 rack_un_collapse_window(struct tcp_rack *rack)
10390 {
10391         struct rack_sendmap *rsm;
10392         int cnt = 0;;
10393
10394         rack->r_ctl.rc_out_at_rto = 0;
10395         rack->r_ctl.rc_snd_max_at_rto = rack->rc_tp->snd_una;
10396         RB_FOREACH_REVERSE(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) {
10397                 if (rsm->r_flags & RACK_RWND_COLLAPSED) {
10398                         rsm->r_flags &= ~RACK_RWND_COLLAPSED;
10399                         rsm->r_flags |= RACK_MUST_RXT;
10400                         if (SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) {
10401                                 rack->r_ctl.rc_snd_max_at_rto = rsm->r_end;
10402                                 rack->r_ctl.rc_out_at_rto += (rsm->r_end - rsm->r_start);
10403                         }
10404                         cnt++;
10405                 }
10406                 else
10407                         break;
10408         }
10409         rack->rc_has_collapsed = 0;
10410         if (cnt) {
10411                 rack->r_must_retran = 1;
10412         }
10413 }
10414
10415 static void
10416 rack_handle_delayed_ack(struct tcpcb *tp, struct tcp_rack *rack,
10417                         int32_t tlen, int32_t tfo_syn)
10418 {
10419         if (DELAY_ACK(tp, tlen) || tfo_syn) {
10420                 if (rack->rc_dack_mode &&
10421                     (tlen > 500) &&
10422                     (rack->rc_dack_toggle == 1)) {
10423                         goto no_delayed_ack;
10424                 }
10425                 rack_timer_cancel(tp, rack,
10426                                   rack->r_ctl.rc_rcvtime, __LINE__);
10427                 tp->t_flags |= TF_DELACK;
10428         } else {
10429 no_delayed_ack:
10430                 rack->r_wanted_output = 1;
10431                 tp->t_flags |= TF_ACKNOW;
10432                 if (rack->rc_dack_mode) {
10433                         if (tp->t_flags & TF_DELACK)
10434                                 rack->rc_dack_toggle = 1;
10435                         else
10436                                 rack->rc_dack_toggle = 0;
10437                 }
10438         }
10439 }
10440
10441 static void
10442 rack_validate_fo_sendwin_up(struct tcpcb *tp, struct tcp_rack *rack)
10443 {
10444         /*
10445          * If fast output is in progress, lets validate that
10446          * the new window did not shrink on us and make it
10447          * so fast output should end.
10448          */
10449         if (rack->r_fast_output) {
10450                 uint32_t out;
10451
10452                 /*
10453                  * Calculate what we will send if left as is
10454                  * and compare that to our send window.
10455                  */
10456                 out = ctf_outstanding(tp);
10457                 if ((out + rack->r_ctl.fsb.left_to_send) > tp->snd_wnd) {
10458                         /* ok we have an issue */
10459                         if (out >= tp->snd_wnd) {
10460                                 /* Turn off fast output the window is met or collapsed */
10461                                 rack->r_fast_output = 0;
10462                         } else {
10463                                 /* we have some room left */
10464                                 rack->r_ctl.fsb.left_to_send = tp->snd_wnd - out;
10465                                 if (rack->r_ctl.fsb.left_to_send < ctf_fixed_maxseg(tp)) {
10466                                         /* If not at least 1 full segment never mind */
10467                                         rack->r_fast_output = 0;
10468                                 }
10469                         }
10470                 }
10471         }
10472 }
10473
10474
10475 /*
10476  * Return value of 1, the TCB is unlocked and most
10477  * likely gone, return value of 0, the TCP is still
10478  * locked.
10479  */
10480 static int
10481 rack_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so,
10482     struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
10483     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
10484 {
10485         /*
10486          * Update window information. Don't look at window if no ACK: TAC's
10487          * send garbage on first SYN.
10488          */
10489         int32_t nsegs;
10490         int32_t tfo_syn;
10491         struct tcp_rack *rack;
10492
10493         rack = (struct tcp_rack *)tp->t_fb_ptr;
10494         INP_WLOCK_ASSERT(tp->t_inpcb);
10495         nsegs = max(1, m->m_pkthdr.lro_nsegs);
10496         if ((thflags & TH_ACK) &&
10497             (SEQ_LT(tp->snd_wl1, th->th_seq) ||
10498             (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||
10499             (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {
10500                 /* keep track of pure window updates */
10501                 if (tlen == 0 &&
10502                     tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)
10503                         KMOD_TCPSTAT_INC(tcps_rcvwinupd);
10504                 tp->snd_wnd = tiwin;
10505                 rack_validate_fo_sendwin_up(tp, rack);
10506                 tp->snd_wl1 = th->th_seq;
10507                 tp->snd_wl2 = th->th_ack;
10508                 if (tp->snd_wnd > tp->max_sndwnd)
10509                         tp->max_sndwnd = tp->snd_wnd;
10510                 rack->r_wanted_output = 1;
10511         } else if (thflags & TH_ACK) {
10512                 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) {
10513                         tp->snd_wnd = tiwin;
10514                         rack_validate_fo_sendwin_up(tp, rack);
10515                         tp->snd_wl1 = th->th_seq;
10516                         tp->snd_wl2 = th->th_ack;
10517                 }
10518         }
10519         if (tp->snd_wnd < ctf_outstanding(tp))
10520                 /* The peer collapsed the window */
10521                 rack_collapsed_window(rack);
10522         else if (rack->rc_has_collapsed)
10523                 rack_un_collapse_window(rack);
10524         /* Was persist timer active and now we have window space? */
10525         if ((rack->rc_in_persist != 0) &&
10526             (tp->snd_wnd >= min((rack->r_ctl.rc_high_rwnd/2),
10527                                 rack->r_ctl.rc_pace_min_segs))) {
10528                 rack_exit_persist(tp, rack, rack->r_ctl.rc_rcvtime);
10529                 tp->snd_nxt = tp->snd_max;
10530                 /* Make sure we output to start the timer */
10531                 rack->r_wanted_output = 1;
10532         }
10533         /* Do we enter persists? */
10534         if ((rack->rc_in_persist == 0) &&
10535             (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), rack->r_ctl.rc_pace_min_segs)) &&
10536             TCPS_HAVEESTABLISHED(tp->t_state) &&
10537             ((tp->snd_max == tp->snd_una) || rack->rc_has_collapsed) &&
10538             sbavail(&tp->t_inpcb->inp_socket->so_snd) &&
10539             (sbavail(&tp->t_inpcb->inp_socket->so_snd) > tp->snd_wnd)) {
10540                 /*
10541                  * Here the rwnd is less than
10542                  * the pacing size, we are established,
10543                  * nothing is outstanding, and there is
10544                  * data to send. Enter persists.
10545                  */
10546                 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime);
10547         }
10548         if (tp->t_flags2 & TF2_DROP_AF_DATA) {
10549                 m_freem(m);
10550                 return (0);
10551         }
10552         /*
10553          * don't process the URG bit, ignore them drag
10554          * along the up.
10555          */
10556         tp->rcv_up = tp->rcv_nxt;
10557         INP_WLOCK_ASSERT(tp->t_inpcb);
10558
10559         /*
10560          * Process the segment text, merging it into the TCP sequencing
10561          * queue, and arranging for acknowledgment of receipt if necessary.
10562          * This process logically involves adjusting tp->rcv_wnd as data is
10563          * presented to the user (this happens in tcp_usrreq.c, case
10564          * PRU_RCVD).  If a FIN has already been received on this connection
10565          * then we just ignore the text.
10566          */
10567         tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&
10568                    IS_FASTOPEN(tp->t_flags));
10569         if ((tlen || (thflags & TH_FIN) || (tfo_syn && tlen > 0)) &&
10570             TCPS_HAVERCVDFIN(tp->t_state) == 0) {
10571                 tcp_seq save_start = th->th_seq;
10572                 tcp_seq save_rnxt  = tp->rcv_nxt;
10573                 int     save_tlen  = tlen;
10574
10575                 m_adj(m, drop_hdrlen);  /* delayed header drop */
10576                 /*
10577                  * Insert segment which includes th into TCP reassembly
10578                  * queue with control block tp.  Set thflags to whether
10579                  * reassembly now includes a segment with FIN.  This handles
10580                  * the common case inline (segment is the next to be
10581                  * received on an established connection, and the queue is
10582                  * empty), avoiding linkage into and removal from the queue
10583                  * and repetition of various conversions. Set DELACK for
10584                  * segments received in order, but ack immediately when
10585                  * segments are out of order (so fast retransmit can work).
10586                  */
10587                 if (th->th_seq == tp->rcv_nxt &&
10588                     SEGQ_EMPTY(tp) &&
10589                     (TCPS_HAVEESTABLISHED(tp->t_state) ||
10590                     tfo_syn)) {
10591 #ifdef NETFLIX_SB_LIMITS
10592                         u_int mcnt, appended;
10593
10594                         if (so->so_rcv.sb_shlim) {
10595                                 mcnt = m_memcnt(m);
10596                                 appended = 0;
10597                                 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
10598                                     CFO_NOSLEEP, NULL) == false) {
10599                                         counter_u64_add(tcp_sb_shlim_fails, 1);
10600                                         m_freem(m);
10601                                         return (0);
10602                                 }
10603                         }
10604 #endif
10605                         rack_handle_delayed_ack(tp, rack, tlen, tfo_syn);
10606                         tp->rcv_nxt += tlen;
10607                         if (tlen &&
10608                             ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
10609                             (tp->t_fbyte_in == 0)) {
10610                                 tp->t_fbyte_in = ticks;
10611                                 if (tp->t_fbyte_in == 0)
10612                                         tp->t_fbyte_in = 1;
10613                                 if (tp->t_fbyte_out && tp->t_fbyte_in)
10614                                         tp->t_flags2 |= TF2_FBYTES_COMPLETE;
10615                         }
10616                         thflags = tcp_get_flags(th) & TH_FIN;
10617                         KMOD_TCPSTAT_ADD(tcps_rcvpack, nsegs);
10618                         KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
10619                         SOCKBUF_LOCK(&so->so_rcv);
10620                         if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
10621                                 m_freem(m);
10622                         } else
10623 #ifdef NETFLIX_SB_LIMITS
10624                                 appended =
10625 #endif
10626                                         sbappendstream_locked(&so->so_rcv, m, 0);
10627
10628                         rack_log_wakeup(tp,rack, &so->so_rcv, tlen, 1);
10629                         /* NB: sorwakeup_locked() does an implicit unlock. */
10630                         sorwakeup_locked(so);
10631 #ifdef NETFLIX_SB_LIMITS
10632                         if (so->so_rcv.sb_shlim && appended != mcnt)
10633                                 counter_fo_release(so->so_rcv.sb_shlim,
10634                                     mcnt - appended);
10635 #endif
10636                 } else {
10637                         /*
10638                          * XXX: Due to the header drop above "th" is
10639                          * theoretically invalid by now.  Fortunately
10640                          * m_adj() doesn't actually frees any mbufs when
10641                          * trimming from the head.
10642                          */
10643                         tcp_seq temp = save_start;
10644
10645                         thflags = tcp_reass(tp, th, &temp, &tlen, m);
10646                         tp->t_flags |= TF_ACKNOW;
10647                         if (tp->t_flags & TF_WAKESOR) {
10648                                 tp->t_flags &= ~TF_WAKESOR;
10649                                 /* NB: sorwakeup_locked() does an implicit unlock. */
10650                                 sorwakeup_locked(so);
10651                         }
10652                 }
10653                 if ((tp->t_flags & TF_SACK_PERMIT) &&
10654                     (save_tlen > 0) &&
10655                     TCPS_HAVEESTABLISHED(tp->t_state)) {
10656                         if ((tlen == 0) && (SEQ_LT(save_start, save_rnxt))) {
10657                                 /*
10658                                  * DSACK actually handled in the fastpath
10659                                  * above.
10660                                  */
10661                                 RACK_OPTS_INC(tcp_sack_path_1);
10662                                 tcp_update_sack_list(tp, save_start,
10663                                     save_start + save_tlen);
10664                         } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) {
10665                                 if ((tp->rcv_numsacks >= 1) &&
10666                                     (tp->sackblks[0].end == save_start)) {
10667                                         /*
10668                                          * Partial overlap, recorded at todrop
10669                                          * above.
10670                                          */
10671                                         RACK_OPTS_INC(tcp_sack_path_2a);
10672                                         tcp_update_sack_list(tp,
10673                                             tp->sackblks[0].start,
10674                                             tp->sackblks[0].end);
10675                                 } else {
10676                                         RACK_OPTS_INC(tcp_sack_path_2b);
10677                                         tcp_update_dsack_list(tp, save_start,
10678                                             save_start + save_tlen);
10679                                 }
10680                         } else if (tlen >= save_tlen) {
10681                                 /* Update of sackblks. */
10682                                 RACK_OPTS_INC(tcp_sack_path_3);
10683                                 tcp_update_dsack_list(tp, save_start,
10684                                     save_start + save_tlen);
10685                         } else if (tlen > 0) {
10686                                 RACK_OPTS_INC(tcp_sack_path_4);
10687                                 tcp_update_dsack_list(tp, save_start,
10688                                     save_start + tlen);
10689                         }
10690                 }
10691         } else {
10692                 m_freem(m);
10693                 thflags &= ~TH_FIN;
10694         }
10695
10696         /*
10697          * If FIN is received ACK the FIN and let the user know that the
10698          * connection is closing.
10699          */
10700         if (thflags & TH_FIN) {
10701                 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {
10702                         /* The socket upcall is handled by socantrcvmore. */
10703                         socantrcvmore(so);
10704                         /*
10705                          * If connection is half-synchronized (ie NEEDSYN
10706                          * flag on) then delay ACK, so it may be piggybacked
10707                          * when SYN is sent. Otherwise, since we received a
10708                          * FIN then no more input can be expected, send ACK
10709                          * now.
10710                          */
10711                         if (tp->t_flags & TF_NEEDSYN) {
10712                                 rack_timer_cancel(tp, rack,
10713                                     rack->r_ctl.rc_rcvtime, __LINE__);
10714                                 tp->t_flags |= TF_DELACK;
10715                         } else {
10716                                 tp->t_flags |= TF_ACKNOW;
10717                         }
10718                         tp->rcv_nxt++;
10719                 }
10720                 switch (tp->t_state) {
10721                         /*
10722                          * In SYN_RECEIVED and ESTABLISHED STATES enter the
10723                          * CLOSE_WAIT state.
10724                          */
10725                 case TCPS_SYN_RECEIVED:
10726                         tp->t_starttime = ticks;
10727                         /* FALLTHROUGH */
10728                 case TCPS_ESTABLISHED:
10729                         rack_timer_cancel(tp, rack,
10730                             rack->r_ctl.rc_rcvtime, __LINE__);
10731                         tcp_state_change(tp, TCPS_CLOSE_WAIT);
10732                         break;
10733
10734                         /*
10735                          * If still in FIN_WAIT_1 STATE FIN has not been
10736                          * acked so enter the CLOSING state.
10737                          */
10738                 case TCPS_FIN_WAIT_1:
10739                         rack_timer_cancel(tp, rack,
10740                             rack->r_ctl.rc_rcvtime, __LINE__);
10741                         tcp_state_change(tp, TCPS_CLOSING);
10742                         break;
10743
10744                         /*
10745                          * In FIN_WAIT_2 state enter the TIME_WAIT state,
10746                          * starting the time-wait timer, turning off the
10747                          * other standard timers.
10748                          */
10749                 case TCPS_FIN_WAIT_2:
10750                         rack_timer_cancel(tp, rack,
10751                             rack->r_ctl.rc_rcvtime, __LINE__);
10752                         tcp_twstart(tp);
10753                         return (1);
10754                 }
10755         }
10756         /*
10757          * Return any desired output.
10758          */
10759         if ((tp->t_flags & TF_ACKNOW) ||
10760             (sbavail(&so->so_snd) > (tp->snd_max - tp->snd_una))) {
10761                 rack->r_wanted_output = 1;
10762         }
10763         INP_WLOCK_ASSERT(tp->t_inpcb);
10764         return (0);
10765 }
10766
10767 /*
10768  * Here nothing is really faster, its just that we
10769  * have broken out the fast-data path also just like
10770  * the fast-ack.
10771  */
10772 static int
10773 rack_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so,
10774     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
10775     uint32_t tiwin, int32_t nxt_pkt, uint8_t iptos)
10776 {
10777         int32_t nsegs;
10778         int32_t newsize = 0;    /* automatic sockbuf scaling */
10779         struct tcp_rack *rack;
10780 #ifdef NETFLIX_SB_LIMITS
10781         u_int mcnt, appended;
10782 #endif
10783 #ifdef TCPDEBUG
10784         /*
10785          * The size of tcp_saveipgen must be the size of the max ip header,
10786          * now IPv6.
10787          */
10788         u_char tcp_saveipgen[IP6_HDR_LEN];
10789         struct tcphdr tcp_savetcp;
10790         short ostate = 0;
10791
10792 #endif
10793         /*
10794          * If last ACK falls within this segment's sequence numbers, record
10795          * the timestamp. NOTE that the test is modified according to the
10796          * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
10797          */
10798         if (__predict_false(th->th_seq != tp->rcv_nxt)) {
10799                 return (0);
10800         }
10801         if (__predict_false(tp->snd_nxt != tp->snd_max)) {
10802                 return (0);
10803         }
10804         if (tiwin && tiwin != tp->snd_wnd) {
10805                 return (0);
10806         }
10807         if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) {
10808                 return (0);
10809         }
10810         if (__predict_false((to->to_flags & TOF_TS) &&
10811             (TSTMP_LT(to->to_tsval, tp->ts_recent)))) {
10812                 return (0);
10813         }
10814         if (__predict_false((th->th_ack != tp->snd_una))) {
10815                 return (0);
10816         }
10817         if (__predict_false(tlen > sbspace(&so->so_rcv))) {
10818                 return (0);
10819         }
10820         if ((to->to_flags & TOF_TS) != 0 &&
10821             SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
10822                 tp->ts_recent_age = tcp_ts_getticks();
10823                 tp->ts_recent = to->to_tsval;
10824         }
10825         rack = (struct tcp_rack *)tp->t_fb_ptr;
10826         /*
10827          * This is a pure, in-sequence data packet with nothing on the
10828          * reassembly queue and we have enough buffer space to take it.
10829          */
10830         nsegs = max(1, m->m_pkthdr.lro_nsegs);
10831
10832 #ifdef NETFLIX_SB_LIMITS
10833         if (so->so_rcv.sb_shlim) {
10834                 mcnt = m_memcnt(m);
10835                 appended = 0;
10836                 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
10837                     CFO_NOSLEEP, NULL) == false) {
10838                         counter_u64_add(tcp_sb_shlim_fails, 1);
10839                         m_freem(m);
10840                         return (1);
10841                 }
10842         }
10843 #endif
10844         /* Clean receiver SACK report if present */
10845         if (tp->rcv_numsacks)
10846                 tcp_clean_sackreport(tp);
10847         KMOD_TCPSTAT_INC(tcps_preddat);
10848         tp->rcv_nxt += tlen;
10849         if (tlen &&
10850             ((tp->t_flags2 & TF2_FBYTES_COMPLETE) == 0) &&
10851             (tp->t_fbyte_in == 0)) {
10852                 tp->t_fbyte_in = ticks;
10853                 if (tp->t_fbyte_in == 0)
10854                         tp->t_fbyte_in = 1;
10855                 if (tp->t_fbyte_out && tp->t_fbyte_in)
10856                         tp->t_flags2 |= TF2_FBYTES_COMPLETE;
10857         }
10858         /*
10859          * Pull snd_wl1 up to prevent seq wrap relative to th_seq.
10860          */
10861         tp->snd_wl1 = th->th_seq;
10862         /*
10863          * Pull rcv_up up to prevent seq wrap relative to rcv_nxt.
10864          */
10865         tp->rcv_up = tp->rcv_nxt;
10866         KMOD_TCPSTAT_ADD(tcps_rcvpack, nsegs);
10867         KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
10868 #ifdef TCPDEBUG
10869         if (so->so_options & SO_DEBUG)
10870                 tcp_trace(TA_INPUT, ostate, tp,
10871                     (void *)tcp_saveipgen, &tcp_savetcp, 0);
10872 #endif
10873         newsize = tcp_autorcvbuf(m, th, so, tp, tlen);
10874
10875         /* Add data to socket buffer. */
10876         SOCKBUF_LOCK(&so->so_rcv);
10877         if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
10878                 m_freem(m);
10879         } else {
10880                 /*
10881                  * Set new socket buffer size. Give up when limit is
10882                  * reached.
10883                  */
10884                 if (newsize)
10885                         if (!sbreserve_locked(so, SO_RCV, newsize, NULL))
10886                                 so->so_rcv.sb_flags &= ~SB_AUTOSIZE;
10887                 m_adj(m, drop_hdrlen);  /* delayed header drop */
10888 #ifdef NETFLIX_SB_LIMITS
10889                 appended =
10890 #endif
10891                         sbappendstream_locked(&so->so_rcv, m, 0);
10892                 ctf_calc_rwin(so, tp);
10893         }
10894         rack_log_wakeup(tp,rack, &so->so_rcv, tlen, 1);
10895         /* NB: sorwakeup_locked() does an implicit unlock. */
10896         sorwakeup_locked(so);
10897 #ifdef NETFLIX_SB_LIMITS
10898         if (so->so_rcv.sb_shlim && mcnt != appended)
10899                 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended);
10900 #endif
10901         rack_handle_delayed_ack(tp, rack, tlen, 0);
10902         if (tp->snd_una == tp->snd_max)
10903                 sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
10904         return (1);
10905 }
10906
10907 /*
10908  * This subfunction is used to try to highly optimize the
10909  * fast path. We again allow window updates that are
10910  * in sequence to remain in the fast-path. We also add
10911  * in the __predict's to attempt to help the compiler.
10912  * Note that if we return a 0, then we can *not* process
10913  * it and the caller should push the packet into the
10914  * slow-path.
10915  */
10916 static int
10917 rack_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
10918     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
10919     uint32_t tiwin, int32_t nxt_pkt, uint32_t cts)
10920 {
10921         int32_t acked;
10922         int32_t nsegs;
10923 #ifdef TCPDEBUG
10924         /*
10925          * The size of tcp_saveipgen must be the size of the max ip header,
10926          * now IPv6.
10927          */
10928         u_char tcp_saveipgen[IP6_HDR_LEN];
10929         struct tcphdr tcp_savetcp;
10930         short ostate = 0;
10931 #endif
10932         int32_t under_pacing = 0;
10933         struct tcp_rack *rack;
10934
10935         if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
10936                 /* Old ack, behind (or duplicate to) the last one rcv'd */
10937                 return (0);
10938         }
10939         if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) {
10940                 /* Above what we have sent? */
10941                 return (0);
10942         }
10943         if (__predict_false(tp->snd_nxt != tp->snd_max)) {
10944                 /* We are retransmitting */
10945                 return (0);
10946         }
10947         if (__predict_false(tiwin == 0)) {
10948                 /* zero window */
10949                 return (0);
10950         }
10951         if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) {
10952                 /* We need a SYN or a FIN, unlikely.. */
10953                 return (0);
10954         }
10955         if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {
10956                 /* Timestamp is behind .. old ack with seq wrap? */
10957                 return (0);
10958         }
10959         if (__predict_false(IN_RECOVERY(tp->t_flags))) {
10960                 /* Still recovering */
10961                 return (0);
10962         }
10963         rack = (struct tcp_rack *)tp->t_fb_ptr;
10964         if (rack->r_ctl.rc_sacked) {
10965                 /* We have sack holes on our scoreboard */
10966                 return (0);
10967         }
10968         /* Ok if we reach here, we can process a fast-ack */
10969         if (rack->gp_ready &&
10970             (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
10971                 under_pacing = 1;
10972         }
10973         nsegs = max(1, m->m_pkthdr.lro_nsegs);
10974         rack_log_ack(tp, to, th, 0, 0);
10975         /* Did the window get updated? */
10976         if (tiwin != tp->snd_wnd) {
10977                 tp->snd_wnd = tiwin;
10978                 rack_validate_fo_sendwin_up(tp, rack);
10979                 tp->snd_wl1 = th->th_seq;
10980                 if (tp->snd_wnd > tp->max_sndwnd)
10981                         tp->max_sndwnd = tp->snd_wnd;
10982         }
10983         /* Do we exit persists? */
10984         if ((rack->rc_in_persist != 0) &&
10985             (tp->snd_wnd >= min((rack->r_ctl.rc_high_rwnd/2),
10986                                rack->r_ctl.rc_pace_min_segs))) {
10987                 rack_exit_persist(tp, rack, cts);
10988         }
10989         /* Do we enter persists? */
10990         if ((rack->rc_in_persist == 0) &&
10991             (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), rack->r_ctl.rc_pace_min_segs)) &&
10992             TCPS_HAVEESTABLISHED(tp->t_state) &&
10993             ((tp->snd_max == tp->snd_una) || rack->rc_has_collapsed) &&
10994             sbavail(&tp->t_inpcb->inp_socket->so_snd) &&
10995             (sbavail(&tp->t_inpcb->inp_socket->so_snd) > tp->snd_wnd)) {
10996                 /*
10997                  * Here the rwnd is less than
10998                  * the pacing size, we are established,
10999                  * nothing is outstanding, and there is
11000                  * data to send. Enter persists.
11001                  */
11002                 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime);
11003         }
11004         /*
11005          * If last ACK falls within this segment's sequence numbers, record
11006          * the timestamp. NOTE that the test is modified according to the
11007          * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
11008          */
11009         if ((to->to_flags & TOF_TS) != 0 &&
11010             SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
11011                 tp->ts_recent_age = tcp_ts_getticks();
11012                 tp->ts_recent = to->to_tsval;
11013         }
11014         /*
11015          * This is a pure ack for outstanding data.
11016          */
11017         KMOD_TCPSTAT_INC(tcps_predack);
11018
11019         /*
11020          * "bad retransmit" recovery.
11021          */
11022         if ((tp->t_flags & TF_PREVVALID) &&
11023             ((tp->t_flags & TF_RCVD_TSTMP) == 0)) {
11024                 tp->t_flags &= ~TF_PREVVALID;
11025                 if (tp->t_rxtshift == 1 &&
11026                     (int)(ticks - tp->t_badrxtwin) < 0)
11027                         rack_cong_signal(tp, CC_RTO_ERR, th->th_ack, __LINE__);
11028         }
11029         /*
11030          * Recalculate the transmit timer / rtt.
11031          *
11032          * Some boxes send broken timestamp replies during the SYN+ACK
11033          * phase, ignore timestamps of 0 or we could calculate a huge RTT
11034          * and blow up the retransmit timer.
11035          */
11036         acked = BYTES_THIS_ACK(tp, th);
11037
11038 #ifdef TCP_HHOOK
11039         /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */
11040         hhook_run_tcp_est_in(tp, th, to);
11041 #endif
11042         KMOD_TCPSTAT_ADD(tcps_rcvackpack, nsegs);
11043         KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
11044         if (acked) {
11045                 struct mbuf *mfree;
11046
11047                 rack_ack_received(tp, rack, th->th_ack, nsegs, CC_ACK, 0);
11048                 SOCKBUF_LOCK(&so->so_snd);
11049                 mfree = sbcut_locked(&so->so_snd, acked);
11050                 tp->snd_una = th->th_ack;
11051                 /* Note we want to hold the sb lock through the sendmap adjust */
11052                 rack_adjust_sendmap(rack, &so->so_snd, tp->snd_una);
11053                 /* Wake up the socket if we have room to write more */
11054                 rack_log_wakeup(tp,rack, &so->so_snd, acked, 2);
11055                 sowwakeup_locked(so);
11056                 m_freem(mfree);
11057                 tp->t_rxtshift = 0;
11058                 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
11059                               rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
11060                 rack->rc_tlp_in_progress = 0;
11061                 rack->r_ctl.rc_tlp_cnt_out = 0;
11062                 /*
11063                  * If it is the RXT timer we want to
11064                  * stop it, so we can restart a TLP.
11065                  */
11066                 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)
11067                         rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
11068 #ifdef NETFLIX_HTTP_LOGGING
11069                 tcp_http_check_for_comp(rack->rc_tp, th->th_ack);
11070 #endif
11071         }
11072         /*
11073          * Let the congestion control algorithm update congestion control
11074          * related information. This typically means increasing the
11075          * congestion window.
11076          */
11077         if (tp->snd_wnd < ctf_outstanding(tp)) {
11078                 /* The peer collapsed the window */
11079                 rack_collapsed_window(rack);
11080         } else if (rack->rc_has_collapsed)
11081                 rack_un_collapse_window(rack);
11082
11083         /*
11084          * Pull snd_wl2 up to prevent seq wrap relative to th_ack.
11085          */
11086         tp->snd_wl2 = th->th_ack;
11087         tp->t_dupacks = 0;
11088         m_freem(m);
11089         /* ND6_HINT(tp);         *//* Some progress has been made. */
11090
11091         /*
11092          * If all outstanding data are acked, stop retransmit timer,
11093          * otherwise restart timer using current (possibly backed-off)
11094          * value. If process is waiting for space, wakeup/selwakeup/signal.
11095          * If data are ready to send, let tcp_output decide between more
11096          * output or persist.
11097          */
11098 #ifdef TCPDEBUG
11099         if (so->so_options & SO_DEBUG)
11100                 tcp_trace(TA_INPUT, ostate, tp,
11101                     (void *)tcp_saveipgen,
11102                     &tcp_savetcp, 0);
11103 #endif
11104         if (under_pacing &&
11105             (rack->use_fixed_rate == 0) &&
11106             (rack->in_probe_rtt == 0) &&
11107             rack->rc_gp_dyn_mul &&
11108             rack->rc_always_pace) {
11109                 /* Check if we are dragging bottom */
11110                 rack_check_bottom_drag(tp, rack, so, acked);
11111         }
11112         if (tp->snd_una == tp->snd_max) {
11113                 tp->t_flags &= ~TF_PREVVALID;
11114                 rack->r_ctl.retran_during_recovery = 0;
11115                 rack->r_ctl.dsack_byte_cnt = 0;
11116                 rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL);
11117                 if (rack->r_ctl.rc_went_idle_time == 0)
11118                         rack->r_ctl.rc_went_idle_time = 1;
11119                 rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__);
11120                 if (sbavail(&tp->t_inpcb->inp_socket->so_snd) == 0)
11121                         tp->t_acktime = 0;
11122                 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
11123         }
11124         if (acked && rack->r_fast_output)
11125                 rack_gain_for_fastoutput(rack, tp, so, (uint32_t)acked);
11126         if (sbavail(&so->so_snd)) {
11127                 rack->r_wanted_output = 1;
11128         }
11129         return (1);
11130 }
11131
11132 /*
11133  * Return value of 1, the TCB is unlocked and most
11134  * likely gone, return value of 0, the TCP is still
11135  * locked.
11136  */
11137 static int
11138 rack_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so,
11139     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
11140     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
11141 {
11142         int32_t ret_val = 0;
11143         int32_t todrop;
11144         int32_t ourfinisacked = 0;
11145         struct tcp_rack *rack;
11146
11147         ctf_calc_rwin(so, tp);
11148         /*
11149          * If the state is SYN_SENT: if seg contains an ACK, but not for our
11150          * SYN, drop the input. if seg contains a RST, then drop the
11151          * connection. if seg does not contain SYN, then drop it. Otherwise
11152          * this is an acceptable SYN segment initialize tp->rcv_nxt and
11153          * tp->irs if seg contains ack then advance tp->snd_una if seg
11154          * contains an ECE and ECN support is enabled, the stream is ECN
11155          * capable. if SYN has been acked change to ESTABLISHED else
11156          * SYN_RCVD state arrange for segment to be acked (eventually)
11157          * continue processing rest of data/controls.
11158          */
11159         if ((thflags & TH_ACK) &&
11160             (SEQ_LEQ(th->th_ack, tp->iss) ||
11161             SEQ_GT(th->th_ack, tp->snd_max))) {
11162                 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
11163                 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
11164                 return (1);
11165         }
11166         if ((thflags & (TH_ACK | TH_RST)) == (TH_ACK | TH_RST)) {
11167                 TCP_PROBE5(connect__refused, NULL, tp,
11168                     mtod(m, const char *), tp, th);
11169                 tp = tcp_drop(tp, ECONNREFUSED);
11170                 ctf_do_drop(m, tp);
11171                 return (1);
11172         }
11173         if (thflags & TH_RST) {
11174                 ctf_do_drop(m, tp);
11175                 return (1);
11176         }
11177         if (!(thflags & TH_SYN)) {
11178                 ctf_do_drop(m, tp);
11179                 return (1);
11180         }
11181         tp->irs = th->th_seq;
11182         tcp_rcvseqinit(tp);
11183         rack = (struct tcp_rack *)tp->t_fb_ptr;
11184         if (thflags & TH_ACK) {
11185                 int tfo_partial = 0;
11186
11187                 KMOD_TCPSTAT_INC(tcps_connects);
11188                 soisconnected(so);
11189 #ifdef MAC
11190                 mac_socketpeer_set_from_mbuf(m, so);
11191 #endif
11192                 /* Do window scaling on this connection? */
11193                 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
11194                     (TF_RCVD_SCALE | TF_REQ_SCALE)) {
11195                         tp->rcv_scale = tp->request_r_scale;
11196                 }
11197                 tp->rcv_adv += min(tp->rcv_wnd,
11198                     TCP_MAXWIN << tp->rcv_scale);
11199                 /*
11200                  * If not all the data that was sent in the TFO SYN
11201                  * has been acked, resend the remainder right away.
11202                  */
11203                 if (IS_FASTOPEN(tp->t_flags) &&
11204                     (tp->snd_una != tp->snd_max)) {
11205                         tp->snd_nxt = th->th_ack;
11206                         tfo_partial = 1;
11207                 }
11208                 /*
11209                  * If there's data, delay ACK; if there's also a FIN ACKNOW
11210                  * will be turned on later.
11211                  */
11212                 if (DELAY_ACK(tp, tlen) && tlen != 0 && !tfo_partial) {
11213                         rack_timer_cancel(tp, rack,
11214                                           rack->r_ctl.rc_rcvtime, __LINE__);
11215                         tp->t_flags |= TF_DELACK;
11216                 } else {
11217                         rack->r_wanted_output = 1;
11218                         tp->t_flags |= TF_ACKNOW;
11219                         rack->rc_dack_toggle = 0;
11220                 }
11221
11222                 tcp_ecn_input_syn_sent(tp, thflags, iptos);
11223
11224                 if (SEQ_GT(th->th_ack, tp->snd_una)) {
11225                         /*
11226                          * We advance snd_una for the
11227                          * fast open case. If th_ack is
11228                          * acknowledging data beyond
11229                          * snd_una we can't just call
11230                          * ack-processing since the
11231                          * data stream in our send-map
11232                          * will start at snd_una + 1 (one
11233                          * beyond the SYN). If its just
11234                          * equal we don't need to do that
11235                          * and there is no send_map.
11236                          */
11237                         tp->snd_una++;
11238                 }
11239                 /*
11240                  * Received <SYN,ACK> in SYN_SENT[*] state. Transitions:
11241                  * SYN_SENT  --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1
11242                  */
11243                 tp->t_starttime = ticks;
11244                 if (tp->t_flags & TF_NEEDFIN) {
11245                         tcp_state_change(tp, TCPS_FIN_WAIT_1);
11246                         tp->t_flags &= ~TF_NEEDFIN;
11247                         thflags &= ~TH_SYN;
11248                 } else {
11249                         tcp_state_change(tp, TCPS_ESTABLISHED);
11250                         TCP_PROBE5(connect__established, NULL, tp,
11251                             mtod(m, const char *), tp, th);
11252                         rack_cc_conn_init(tp);
11253                 }
11254         } else {
11255                 /*
11256                  * Received initial SYN in SYN-SENT[*] state => simultaneous
11257                  * open.  If segment contains CC option and there is a
11258                  * cached CC, apply TAO test. If it succeeds, connection is *
11259                  * half-synchronized. Otherwise, do 3-way handshake:
11260                  * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If
11261                  * there was no CC option, clear cached CC value.
11262                  */
11263                 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN);
11264                 tcp_state_change(tp, TCPS_SYN_RECEIVED);
11265         }
11266         INP_WLOCK_ASSERT(tp->t_inpcb);
11267         /*
11268          * Advance th->th_seq to correspond to first data byte. If data,
11269          * trim to stay within window, dropping FIN if necessary.
11270          */
11271         th->th_seq++;
11272         if (tlen > tp->rcv_wnd) {
11273                 todrop = tlen - tp->rcv_wnd;
11274                 m_adj(m, -todrop);
11275                 tlen = tp->rcv_wnd;
11276                 thflags &= ~TH_FIN;
11277                 KMOD_TCPSTAT_INC(tcps_rcvpackafterwin);
11278                 KMOD_TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop);
11279         }
11280         tp->snd_wl1 = th->th_seq - 1;
11281         tp->rcv_up = th->th_seq;
11282         /*
11283          * Client side of transaction: already sent SYN and data. If the
11284          * remote host used T/TCP to validate the SYN, our data will be
11285          * ACK'd; if so, enter normal data segment processing in the middle
11286          * of step 5, ack processing. Otherwise, goto step 6.
11287          */
11288         if (thflags & TH_ACK) {
11289                 /* For syn-sent we need to possibly update the rtt */
11290                 if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) {
11291                         uint32_t t, mcts;
11292
11293                         mcts = tcp_ts_getticks();
11294                         t = (mcts - to->to_tsecr) * HPTS_USEC_IN_MSEC;
11295                         if (!tp->t_rttlow || tp->t_rttlow > t)
11296                                 tp->t_rttlow = t;
11297                         rack_log_rtt_sample_calc(rack, t, (to->to_tsecr * 1000), (mcts * 1000), 4);
11298                         tcp_rack_xmit_timer(rack, t + 1, 1, t, 0, NULL, 2);
11299                         tcp_rack_xmit_timer_commit(rack, tp);
11300                 }
11301                 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val))
11302                         return (ret_val);
11303                 /* We may have changed to FIN_WAIT_1 above */
11304                 if (tp->t_state == TCPS_FIN_WAIT_1) {
11305                         /*
11306                          * In FIN_WAIT_1 STATE in addition to the processing
11307                          * for the ESTABLISHED state if our FIN is now
11308                          * acknowledged then enter FIN_WAIT_2.
11309                          */
11310                         if (ourfinisacked) {
11311                                 /*
11312                                  * If we can't receive any more data, then
11313                                  * closing user can proceed. Starting the
11314                                  * timer is contrary to the specification,
11315                                  * but if we don't get a FIN we'll hang
11316                                  * forever.
11317                                  *
11318                                  * XXXjl: we should release the tp also, and
11319                                  * use a compressed state.
11320                                  */
11321                                 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
11322                                         soisdisconnected(so);
11323                                         tcp_timer_activate(tp, TT_2MSL,
11324                                             (tcp_fast_finwait2_recycle ?
11325                                             tcp_finwait2_timeout :
11326                                             TP_MAXIDLE(tp)));
11327                                 }
11328                                 tcp_state_change(tp, TCPS_FIN_WAIT_2);
11329                         }
11330                 }
11331         }
11332         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
11333            tiwin, thflags, nxt_pkt));
11334 }
11335
11336 /*
11337  * Return value of 1, the TCB is unlocked and most
11338  * likely gone, return value of 0, the TCP is still
11339  * locked.
11340  */
11341 static int
11342 rack_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so,
11343     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
11344     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
11345 {
11346         struct tcp_rack *rack;
11347         int32_t ret_val = 0;
11348         int32_t ourfinisacked = 0;
11349
11350         ctf_calc_rwin(so, tp);
11351         if ((thflags & TH_ACK) &&
11352             (SEQ_LEQ(th->th_ack, tp->snd_una) ||
11353             SEQ_GT(th->th_ack, tp->snd_max))) {
11354                 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
11355                 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
11356                 return (1);
11357         }
11358         rack = (struct tcp_rack *)tp->t_fb_ptr;
11359         if (IS_FASTOPEN(tp->t_flags)) {
11360                 /*
11361                  * When a TFO connection is in SYN_RECEIVED, the
11362                  * only valid packets are the initial SYN, a
11363                  * retransmit/copy of the initial SYN (possibly with
11364                  * a subset of the original data), a valid ACK, a
11365                  * FIN, or a RST.
11366                  */
11367                 if ((thflags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
11368                         tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
11369                         ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
11370                         return (1);
11371                 } else if (thflags & TH_SYN) {
11372                         /* non-initial SYN is ignored */
11373                         if ((rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT) ||
11374                             (rack->r_ctl.rc_hpts_flags & PACE_TMR_TLP) ||
11375                             (rack->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) {
11376                                 ctf_do_drop(m, NULL);
11377                                 return (0);
11378                         }
11379                 } else if (!(thflags & (TH_ACK | TH_FIN | TH_RST))) {
11380                         ctf_do_drop(m, NULL);
11381                         return (0);
11382                 }
11383         }
11384
11385         if ((thflags & TH_RST) ||
11386             (tp->t_fin_is_rst && (thflags & TH_FIN)))
11387                 return (__ctf_process_rst(m, th, so, tp,
11388                                           &rack->r_ctl.challenge_ack_ts,
11389                                           &rack->r_ctl.challenge_ack_cnt));
11390         /*
11391          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
11392          * it's less than ts_recent, drop it.
11393          */
11394         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
11395             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
11396                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
11397                         return (ret_val);
11398         }
11399         /*
11400          * In the SYN-RECEIVED state, validate that the packet belongs to
11401          * this connection before trimming the data to fit the receive
11402          * window.  Check the sequence number versus IRS since we know the
11403          * sequence numbers haven't wrapped.  This is a partial fix for the
11404          * "LAND" DoS attack.
11405          */
11406         if (SEQ_LT(th->th_seq, tp->irs)) {
11407                 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
11408                 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
11409                 return (1);
11410         }
11411         if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val,
11412                               &rack->r_ctl.challenge_ack_ts,
11413                               &rack->r_ctl.challenge_ack_cnt)) {
11414                 return (ret_val);
11415         }
11416         /*
11417          * If last ACK falls within this segment's sequence numbers, record
11418          * its timestamp. NOTE: 1) That the test incorporates suggestions
11419          * from the latest proposal of the tcplw@cray.com list (Braden
11420          * 1993/04/26). 2) That updating only on newer timestamps interferes
11421          * with our earlier PAWS tests, so this check should be solely
11422          * predicated on the sequence space of this segment. 3) That we
11423          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
11424          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
11425          * SEG.Len, This modified check allows us to overcome RFC1323's
11426          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
11427          * p.869. In such cases, we can still calculate the RTT correctly
11428          * when RCV.NXT == Last.ACK.Sent.
11429          */
11430         if ((to->to_flags & TOF_TS) != 0 &&
11431             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
11432             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
11433             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
11434                 tp->ts_recent_age = tcp_ts_getticks();
11435                 tp->ts_recent = to->to_tsval;
11436         }
11437         tp->snd_wnd = tiwin;
11438         rack_validate_fo_sendwin_up(tp, rack);
11439         /*
11440          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
11441          * is on (half-synchronized state), then queue data for later
11442          * processing; else drop segment and return.
11443          */
11444         if ((thflags & TH_ACK) == 0) {
11445                 if (IS_FASTOPEN(tp->t_flags)) {
11446                         rack_cc_conn_init(tp);
11447                 }
11448                 return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
11449                     tiwin, thflags, nxt_pkt));
11450         }
11451         KMOD_TCPSTAT_INC(tcps_connects);
11452         soisconnected(so);
11453         /* Do window scaling? */
11454         if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
11455             (TF_RCVD_SCALE | TF_REQ_SCALE)) {
11456                 tp->rcv_scale = tp->request_r_scale;
11457         }
11458         /*
11459          * Make transitions: SYN-RECEIVED  -> ESTABLISHED SYN-RECEIVED* ->
11460          * FIN-WAIT-1
11461          */
11462         tp->t_starttime = ticks;
11463         if (IS_FASTOPEN(tp->t_flags) && tp->t_tfo_pending) {
11464                 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
11465                 tp->t_tfo_pending = NULL;
11466         }
11467         if (tp->t_flags & TF_NEEDFIN) {
11468                 tcp_state_change(tp, TCPS_FIN_WAIT_1);
11469                 tp->t_flags &= ~TF_NEEDFIN;
11470         } else {
11471                 tcp_state_change(tp, TCPS_ESTABLISHED);
11472                 TCP_PROBE5(accept__established, NULL, tp,
11473                     mtod(m, const char *), tp, th);
11474                 /*
11475                  * TFO connections call cc_conn_init() during SYN
11476                  * processing.  Calling it again here for such connections
11477                  * is not harmless as it would undo the snd_cwnd reduction
11478                  * that occurs when a TFO SYN|ACK is retransmitted.
11479                  */
11480                 if (!IS_FASTOPEN(tp->t_flags))
11481                         rack_cc_conn_init(tp);
11482         }
11483         /*
11484          * Account for the ACK of our SYN prior to
11485          * regular ACK processing below, except for
11486          * simultaneous SYN, which is handled later.
11487          */
11488         if (SEQ_GT(th->th_ack, tp->snd_una) && !(tp->t_flags & TF_NEEDSYN))
11489                 tp->snd_una++;
11490         /*
11491          * If segment contains data or ACK, will call tcp_reass() later; if
11492          * not, do so now to pass queued data to user.
11493          */
11494         if (tlen == 0 && (thflags & TH_FIN) == 0) {
11495                 (void) tcp_reass(tp, (struct tcphdr *)0, NULL, 0,
11496                     (struct mbuf *)0);
11497                 if (tp->t_flags & TF_WAKESOR) {
11498                         tp->t_flags &= ~TF_WAKESOR;
11499                         /* NB: sorwakeup_locked() does an implicit unlock. */
11500                         sorwakeup_locked(so);
11501                 }
11502         }
11503         tp->snd_wl1 = th->th_seq - 1;
11504         /* For syn-recv we need to possibly update the rtt */
11505         if ((to->to_flags & TOF_TS) != 0 && to->to_tsecr) {
11506                 uint32_t t, mcts;
11507
11508                 mcts = tcp_ts_getticks();
11509                 t = (mcts - to->to_tsecr) * HPTS_USEC_IN_MSEC;
11510                 if (!tp->t_rttlow || tp->t_rttlow > t)
11511                         tp->t_rttlow = t;
11512                 rack_log_rtt_sample_calc(rack, t, (to->to_tsecr * 1000), (mcts * 1000), 5);
11513                 tcp_rack_xmit_timer(rack, t + 1, 1, t, 0, NULL, 2);
11514                 tcp_rack_xmit_timer_commit(rack, tp);
11515         }
11516         if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
11517                 return (ret_val);
11518         }
11519         if (tp->t_state == TCPS_FIN_WAIT_1) {
11520                 /* We could have went to FIN_WAIT_1 (or EST) above */
11521                 /*
11522                  * In FIN_WAIT_1 STATE in addition to the processing for the
11523                  * ESTABLISHED state if our FIN is now acknowledged then
11524                  * enter FIN_WAIT_2.
11525                  */
11526                 if (ourfinisacked) {
11527                         /*
11528                          * If we can't receive any more data, then closing
11529                          * user can proceed. Starting the timer is contrary
11530                          * to the specification, but if we don't get a FIN
11531                          * we'll hang forever.
11532                          *
11533                          * XXXjl: we should release the tp also, and use a
11534                          * compressed state.
11535                          */
11536                         if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
11537                                 soisdisconnected(so);
11538                                 tcp_timer_activate(tp, TT_2MSL,
11539                                     (tcp_fast_finwait2_recycle ?
11540                                     tcp_finwait2_timeout :
11541                                     TP_MAXIDLE(tp)));
11542                         }
11543                         tcp_state_change(tp, TCPS_FIN_WAIT_2);
11544                 }
11545         }
11546         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
11547             tiwin, thflags, nxt_pkt));
11548 }
11549
11550 /*
11551  * Return value of 1, the TCB is unlocked and most
11552  * likely gone, return value of 0, the TCP is still
11553  * locked.
11554  */
11555 static int
11556 rack_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so,
11557     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
11558     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
11559 {
11560         int32_t ret_val = 0;
11561         struct tcp_rack *rack;
11562
11563         /*
11564          * Header prediction: check for the two common cases of a
11565          * uni-directional data xfer.  If the packet has no control flags,
11566          * is in-sequence, the window didn't change and we're not
11567          * retransmitting, it's a candidate.  If the length is zero and the
11568          * ack moved forward, we're the sender side of the xfer.  Just free
11569          * the data acked & wake any higher level process that was blocked
11570          * waiting for space.  If the length is non-zero and the ack didn't
11571          * move, we're the receiver side.  If we're getting packets in-order
11572          * (the reassembly queue is empty), add the data toc The socket
11573          * buffer and note that we need a delayed ack. Make sure that the
11574          * hidden state-flags are also off. Since we check for
11575          * TCPS_ESTABLISHED first, it can only be TH_NEEDSYN.
11576          */
11577         rack = (struct tcp_rack *)tp->t_fb_ptr;
11578         if (__predict_true(((to->to_flags & TOF_SACK) == 0)) &&
11579             __predict_true((thflags & (TH_SYN | TH_FIN | TH_RST | TH_ACK)) == TH_ACK) &&
11580             __predict_true(SEGQ_EMPTY(tp)) &&
11581             __predict_true(th->th_seq == tp->rcv_nxt)) {
11582                 if (tlen == 0) {
11583                         if (rack_fastack(m, th, so, tp, to, drop_hdrlen, tlen,
11584                             tiwin, nxt_pkt, rack->r_ctl.rc_rcvtime)) {
11585                                 return (0);
11586                         }
11587                 } else {
11588                         if (rack_do_fastnewdata(m, th, so, tp, to, drop_hdrlen, tlen,
11589                             tiwin, nxt_pkt, iptos)) {
11590                                 return (0);
11591                         }
11592                 }
11593         }
11594         ctf_calc_rwin(so, tp);
11595
11596         if ((thflags & TH_RST) ||
11597             (tp->t_fin_is_rst && (thflags & TH_FIN)))
11598                 return (__ctf_process_rst(m, th, so, tp,
11599                                           &rack->r_ctl.challenge_ack_ts,
11600                                           &rack->r_ctl.challenge_ack_cnt));
11601
11602         /*
11603          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
11604          * synchronized state.
11605          */
11606         if (thflags & TH_SYN) {
11607                 ctf_challenge_ack(m, th, tp, &ret_val);
11608                 return (ret_val);
11609         }
11610         /*
11611          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
11612          * it's less than ts_recent, drop it.
11613          */
11614         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
11615             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
11616                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
11617                         return (ret_val);
11618         }
11619         if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val,
11620                               &rack->r_ctl.challenge_ack_ts,
11621                               &rack->r_ctl.challenge_ack_cnt)) {
11622                 return (ret_val);
11623         }
11624         /*
11625          * If last ACK falls within this segment's sequence numbers, record
11626          * its timestamp. NOTE: 1) That the test incorporates suggestions
11627          * from the latest proposal of the tcplw@cray.com list (Braden
11628          * 1993/04/26). 2) That updating only on newer timestamps interferes
11629          * with our earlier PAWS tests, so this check should be solely
11630          * predicated on the sequence space of this segment. 3) That we
11631          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
11632          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
11633          * SEG.Len, This modified check allows us to overcome RFC1323's
11634          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
11635          * p.869. In such cases, we can still calculate the RTT correctly
11636          * when RCV.NXT == Last.ACK.Sent.
11637          */
11638         if ((to->to_flags & TOF_TS) != 0 &&
11639             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
11640             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
11641             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
11642                 tp->ts_recent_age = tcp_ts_getticks();
11643                 tp->ts_recent = to->to_tsval;
11644         }
11645         /*
11646          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
11647          * is on (half-synchronized state), then queue data for later
11648          * processing; else drop segment and return.
11649          */
11650         if ((thflags & TH_ACK) == 0) {
11651                 if (tp->t_flags & TF_NEEDSYN) {
11652                         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
11653                             tiwin, thflags, nxt_pkt));
11654
11655                 } else if (tp->t_flags & TF_ACKNOW) {
11656                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
11657                         ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
11658                         return (ret_val);
11659                 } else {
11660                         ctf_do_drop(m, NULL);
11661                         return (0);
11662                 }
11663         }
11664         /*
11665          * Ack processing.
11666          */
11667         if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
11668                 return (ret_val);
11669         }
11670         if (sbavail(&so->so_snd)) {
11671                 if (ctf_progress_timeout_check(tp, true)) {
11672                         rack_log_progress_event(rack, tp, tick, PROGRESS_DROP, __LINE__);
11673                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
11674                         return (1);
11675                 }
11676         }
11677         /* State changes only happen in rack_process_data() */
11678         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
11679             tiwin, thflags, nxt_pkt));
11680 }
11681
11682 /*
11683  * Return value of 1, the TCB is unlocked and most
11684  * likely gone, return value of 0, the TCP is still
11685  * locked.
11686  */
11687 static int
11688 rack_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so,
11689     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
11690     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
11691 {
11692         int32_t ret_val = 0;
11693         struct tcp_rack *rack;
11694
11695         rack = (struct tcp_rack *)tp->t_fb_ptr;
11696         ctf_calc_rwin(so, tp);
11697         if ((thflags & TH_RST) ||
11698             (tp->t_fin_is_rst && (thflags & TH_FIN)))
11699                 return (__ctf_process_rst(m, th, so, tp,
11700                                           &rack->r_ctl.challenge_ack_ts,
11701                                           &rack->r_ctl.challenge_ack_cnt));
11702         /*
11703          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
11704          * synchronized state.
11705          */
11706         if (thflags & TH_SYN) {
11707                 ctf_challenge_ack(m, th, tp, &ret_val);
11708                 return (ret_val);
11709         }
11710         /*
11711          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
11712          * it's less than ts_recent, drop it.
11713          */
11714         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
11715             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
11716                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
11717                         return (ret_val);
11718         }
11719         if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val,
11720                               &rack->r_ctl.challenge_ack_ts,
11721                               &rack->r_ctl.challenge_ack_cnt)) {
11722                 return (ret_val);
11723         }
11724         /*
11725          * If last ACK falls within this segment's sequence numbers, record
11726          * its timestamp. NOTE: 1) That the test incorporates suggestions
11727          * from the latest proposal of the tcplw@cray.com list (Braden
11728          * 1993/04/26). 2) That updating only on newer timestamps interferes
11729          * with our earlier PAWS tests, so this check should be solely
11730          * predicated on the sequence space of this segment. 3) That we
11731          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
11732          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
11733          * SEG.Len, This modified check allows us to overcome RFC1323's
11734          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
11735          * p.869. In such cases, we can still calculate the RTT correctly
11736          * when RCV.NXT == Last.ACK.Sent.
11737          */
11738         if ((to->to_flags & TOF_TS) != 0 &&
11739             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
11740             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
11741             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
11742                 tp->ts_recent_age = tcp_ts_getticks();
11743                 tp->ts_recent = to->to_tsval;
11744         }
11745         /*
11746          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
11747          * is on (half-synchronized state), then queue data for later
11748          * processing; else drop segment and return.
11749          */
11750         if ((thflags & TH_ACK) == 0) {
11751                 if (tp->t_flags & TF_NEEDSYN) {
11752                         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
11753                             tiwin, thflags, nxt_pkt));
11754
11755                 } else if (tp->t_flags & TF_ACKNOW) {
11756                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
11757                         ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
11758                         return (ret_val);
11759                 } else {
11760                         ctf_do_drop(m, NULL);
11761                         return (0);
11762                 }
11763         }
11764         /*
11765          * Ack processing.
11766          */
11767         if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
11768                 return (ret_val);
11769         }
11770         if (sbavail(&so->so_snd)) {
11771                 if (ctf_progress_timeout_check(tp, true)) {
11772                         rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
11773                                                 tp, tick, PROGRESS_DROP, __LINE__);
11774                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
11775                         return (1);
11776                 }
11777         }
11778         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
11779             tiwin, thflags, nxt_pkt));
11780 }
11781
11782 static int
11783 rack_check_data_after_close(struct mbuf *m,
11784     struct tcpcb *tp, int32_t *tlen, struct tcphdr *th, struct socket *so)
11785 {
11786         struct tcp_rack *rack;
11787
11788         rack = (struct tcp_rack *)tp->t_fb_ptr;
11789         if (rack->rc_allow_data_af_clo == 0) {
11790         close_now:
11791                 tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE);
11792                 /* tcp_close will kill the inp pre-log the Reset */
11793                 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
11794                 tp = tcp_close(tp);
11795                 KMOD_TCPSTAT_INC(tcps_rcvafterclose);
11796                 ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, (*tlen));
11797                 return (1);
11798         }
11799         if (sbavail(&so->so_snd) == 0)
11800                 goto close_now;
11801         /* Ok we allow data that is ignored and a followup reset */
11802         tcp_log_end_status(tp, TCP_EI_STATUS_DATA_A_CLOSE);
11803         tp->rcv_nxt = th->th_seq + *tlen;
11804         tp->t_flags2 |= TF2_DROP_AF_DATA;
11805         rack->r_wanted_output = 1;
11806         *tlen = 0;
11807         return (0);
11808 }
11809
11810 /*
11811  * Return value of 1, the TCB is unlocked and most
11812  * likely gone, return value of 0, the TCP is still
11813  * locked.
11814  */
11815 static int
11816 rack_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so,
11817     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
11818     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
11819 {
11820         int32_t ret_val = 0;
11821         int32_t ourfinisacked = 0;
11822         struct tcp_rack *rack;
11823
11824         rack = (struct tcp_rack *)tp->t_fb_ptr;
11825         ctf_calc_rwin(so, tp);
11826
11827         if ((thflags & TH_RST) ||
11828             (tp->t_fin_is_rst && (thflags & TH_FIN)))
11829                 return (__ctf_process_rst(m, th, so, tp,
11830                                           &rack->r_ctl.challenge_ack_ts,
11831                                           &rack->r_ctl.challenge_ack_cnt));
11832         /*
11833          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
11834          * synchronized state.
11835          */
11836         if (thflags & TH_SYN) {
11837                 ctf_challenge_ack(m, th, tp, &ret_val);
11838                 return (ret_val);
11839         }
11840         /*
11841          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
11842          * it's less than ts_recent, drop it.
11843          */
11844         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
11845             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
11846                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
11847                         return (ret_val);
11848         }
11849         if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val,
11850                               &rack->r_ctl.challenge_ack_ts,
11851                               &rack->r_ctl.challenge_ack_cnt)) {
11852                 return (ret_val);
11853         }
11854         /*
11855          * If new data are received on a connection after the user processes
11856          * are gone, then RST the other end.
11857          */
11858         if ((tp->t_flags & TF_CLOSED) && tlen &&
11859             rack_check_data_after_close(m, tp, &tlen, th, so))
11860                 return (1);
11861         /*
11862          * If last ACK falls within this segment's sequence numbers, record
11863          * its timestamp. NOTE: 1) That the test incorporates suggestions
11864          * from the latest proposal of the tcplw@cray.com list (Braden
11865          * 1993/04/26). 2) That updating only on newer timestamps interferes
11866          * with our earlier PAWS tests, so this check should be solely
11867          * predicated on the sequence space of this segment. 3) That we
11868          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
11869          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
11870          * SEG.Len, This modified check allows us to overcome RFC1323's
11871          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
11872          * p.869. In such cases, we can still calculate the RTT correctly
11873          * when RCV.NXT == Last.ACK.Sent.
11874          */
11875         if ((to->to_flags & TOF_TS) != 0 &&
11876             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
11877             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
11878             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
11879                 tp->ts_recent_age = tcp_ts_getticks();
11880                 tp->ts_recent = to->to_tsval;
11881         }
11882         /*
11883          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
11884          * is on (half-synchronized state), then queue data for later
11885          * processing; else drop segment and return.
11886          */
11887         if ((thflags & TH_ACK) == 0) {
11888                 if (tp->t_flags & TF_NEEDSYN) {
11889                         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
11890                             tiwin, thflags, nxt_pkt));
11891                 } else if (tp->t_flags & TF_ACKNOW) {
11892                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
11893                         ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
11894                         return (ret_val);
11895                 } else {
11896                         ctf_do_drop(m, NULL);
11897                         return (0);
11898                 }
11899         }
11900         /*
11901          * Ack processing.
11902          */
11903         if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
11904                 return (ret_val);
11905         }
11906         if (ourfinisacked) {
11907                 /*
11908                  * If we can't receive any more data, then closing user can
11909                  * proceed. Starting the timer is contrary to the
11910                  * specification, but if we don't get a FIN we'll hang
11911                  * forever.
11912                  *
11913                  * XXXjl: we should release the tp also, and use a
11914                  * compressed state.
11915                  */
11916                 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
11917                         soisdisconnected(so);
11918                         tcp_timer_activate(tp, TT_2MSL,
11919                             (tcp_fast_finwait2_recycle ?
11920                             tcp_finwait2_timeout :
11921                             TP_MAXIDLE(tp)));
11922                 }
11923                 tcp_state_change(tp, TCPS_FIN_WAIT_2);
11924         }
11925         if (sbavail(&so->so_snd)) {
11926                 if (ctf_progress_timeout_check(tp, true)) {
11927                         rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
11928                                                 tp, tick, PROGRESS_DROP, __LINE__);
11929                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
11930                         return (1);
11931                 }
11932         }
11933         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
11934             tiwin, thflags, nxt_pkt));
11935 }
11936
11937 /*
11938  * Return value of 1, the TCB is unlocked and most
11939  * likely gone, return value of 0, the TCP is still
11940  * locked.
11941  */
11942 static int
11943 rack_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so,
11944     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
11945     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
11946 {
11947         int32_t ret_val = 0;
11948         int32_t ourfinisacked = 0;
11949         struct tcp_rack *rack;
11950
11951         rack = (struct tcp_rack *)tp->t_fb_ptr;
11952         ctf_calc_rwin(so, tp);
11953
11954         if ((thflags & TH_RST) ||
11955             (tp->t_fin_is_rst && (thflags & TH_FIN)))
11956                 return (__ctf_process_rst(m, th, so, tp,
11957                                           &rack->r_ctl.challenge_ack_ts,
11958                                           &rack->r_ctl.challenge_ack_cnt));
11959         /*
11960          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
11961          * synchronized state.
11962          */
11963         if (thflags & TH_SYN) {
11964                 ctf_challenge_ack(m, th, tp, &ret_val);
11965                 return (ret_val);
11966         }
11967         /*
11968          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
11969          * it's less than ts_recent, drop it.
11970          */
11971         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
11972             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
11973                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
11974                         return (ret_val);
11975         }
11976         if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val,
11977                               &rack->r_ctl.challenge_ack_ts,
11978                               &rack->r_ctl.challenge_ack_cnt)) {
11979                 return (ret_val);
11980         }
11981         /*
11982          * If new data are received on a connection after the user processes
11983          * are gone, then RST the other end.
11984          */
11985         if ((tp->t_flags & TF_CLOSED) && tlen &&
11986             rack_check_data_after_close(m, tp, &tlen, th, so))
11987                 return (1);
11988         /*
11989          * If last ACK falls within this segment's sequence numbers, record
11990          * its timestamp. NOTE: 1) That the test incorporates suggestions
11991          * from the latest proposal of the tcplw@cray.com list (Braden
11992          * 1993/04/26). 2) That updating only on newer timestamps interferes
11993          * with our earlier PAWS tests, so this check should be solely
11994          * predicated on the sequence space of this segment. 3) That we
11995          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
11996          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
11997          * SEG.Len, This modified check allows us to overcome RFC1323's
11998          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
11999          * p.869. In such cases, we can still calculate the RTT correctly
12000          * when RCV.NXT == Last.ACK.Sent.
12001          */
12002         if ((to->to_flags & TOF_TS) != 0 &&
12003             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
12004             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
12005             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
12006                 tp->ts_recent_age = tcp_ts_getticks();
12007                 tp->ts_recent = to->to_tsval;
12008         }
12009         /*
12010          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
12011          * is on (half-synchronized state), then queue data for later
12012          * processing; else drop segment and return.
12013          */
12014         if ((thflags & TH_ACK) == 0) {
12015                 if (tp->t_flags & TF_NEEDSYN) {
12016                         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
12017                             tiwin, thflags, nxt_pkt));
12018                 } else if (tp->t_flags & TF_ACKNOW) {
12019                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
12020                         ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
12021                         return (ret_val);
12022                 } else {
12023                         ctf_do_drop(m, NULL);
12024                         return (0);
12025                 }
12026         }
12027         /*
12028          * Ack processing.
12029          */
12030         if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
12031                 return (ret_val);
12032         }
12033         if (ourfinisacked) {
12034                 tcp_twstart(tp);
12035                 m_freem(m);
12036                 return (1);
12037         }
12038         if (sbavail(&so->so_snd)) {
12039                 if (ctf_progress_timeout_check(tp, true)) {
12040                         rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
12041                                                 tp, tick, PROGRESS_DROP, __LINE__);
12042                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
12043                         return (1);
12044                 }
12045         }
12046         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
12047             tiwin, thflags, nxt_pkt));
12048 }
12049
12050 /*
12051  * Return value of 1, the TCB is unlocked and most
12052  * likely gone, return value of 0, the TCP is still
12053  * locked.
12054  */
12055 static int
12056 rack_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
12057     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
12058     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
12059 {
12060         int32_t ret_val = 0;
12061         int32_t ourfinisacked = 0;
12062         struct tcp_rack *rack;
12063
12064         rack = (struct tcp_rack *)tp->t_fb_ptr;
12065         ctf_calc_rwin(so, tp);
12066
12067         if ((thflags & TH_RST) ||
12068             (tp->t_fin_is_rst && (thflags & TH_FIN)))
12069                 return (__ctf_process_rst(m, th, so, tp,
12070                                           &rack->r_ctl.challenge_ack_ts,
12071                                           &rack->r_ctl.challenge_ack_cnt));
12072         /*
12073          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
12074          * synchronized state.
12075          */
12076         if (thflags & TH_SYN) {
12077                 ctf_challenge_ack(m, th, tp, &ret_val);
12078                 return (ret_val);
12079         }
12080         /*
12081          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
12082          * it's less than ts_recent, drop it.
12083          */
12084         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
12085             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
12086                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
12087                         return (ret_val);
12088         }
12089         if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val,
12090                               &rack->r_ctl.challenge_ack_ts,
12091                               &rack->r_ctl.challenge_ack_cnt)) {
12092                 return (ret_val);
12093         }
12094         /*
12095          * If new data are received on a connection after the user processes
12096          * are gone, then RST the other end.
12097          */
12098         if ((tp->t_flags & TF_CLOSED) && tlen &&
12099             rack_check_data_after_close(m, tp, &tlen, th, so))
12100                 return (1);
12101         /*
12102          * If last ACK falls within this segment's sequence numbers, record
12103          * its timestamp. NOTE: 1) That the test incorporates suggestions
12104          * from the latest proposal of the tcplw@cray.com list (Braden
12105          * 1993/04/26). 2) That updating only on newer timestamps interferes
12106          * with our earlier PAWS tests, so this check should be solely
12107          * predicated on the sequence space of this segment. 3) That we
12108          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
12109          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
12110          * SEG.Len, This modified check allows us to overcome RFC1323's
12111          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
12112          * p.869. In such cases, we can still calculate the RTT correctly
12113          * when RCV.NXT == Last.ACK.Sent.
12114          */
12115         if ((to->to_flags & TOF_TS) != 0 &&
12116             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
12117             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
12118             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
12119                 tp->ts_recent_age = tcp_ts_getticks();
12120                 tp->ts_recent = to->to_tsval;
12121         }
12122         /*
12123          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
12124          * is on (half-synchronized state), then queue data for later
12125          * processing; else drop segment and return.
12126          */
12127         if ((thflags & TH_ACK) == 0) {
12128                 if (tp->t_flags & TF_NEEDSYN) {
12129                         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
12130                             tiwin, thflags, nxt_pkt));
12131                 } else if (tp->t_flags & TF_ACKNOW) {
12132                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
12133                         ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
12134                         return (ret_val);
12135                 } else {
12136                         ctf_do_drop(m, NULL);
12137                         return (0);
12138                 }
12139         }
12140         /*
12141          * case TCPS_LAST_ACK: Ack processing.
12142          */
12143         if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
12144                 return (ret_val);
12145         }
12146         if (ourfinisacked) {
12147                 tp = tcp_close(tp);
12148                 ctf_do_drop(m, tp);
12149                 return (1);
12150         }
12151         if (sbavail(&so->so_snd)) {
12152                 if (ctf_progress_timeout_check(tp, true)) {
12153                         rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
12154                                                 tp, tick, PROGRESS_DROP, __LINE__);
12155                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
12156                         return (1);
12157                 }
12158         }
12159         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
12160             tiwin, thflags, nxt_pkt));
12161 }
12162
12163 /*
12164  * Return value of 1, the TCB is unlocked and most
12165  * likely gone, return value of 0, the TCP is still
12166  * locked.
12167  */
12168 static int
12169 rack_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so,
12170     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
12171     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt, uint8_t iptos)
12172 {
12173         int32_t ret_val = 0;
12174         int32_t ourfinisacked = 0;
12175         struct tcp_rack *rack;
12176
12177         rack = (struct tcp_rack *)tp->t_fb_ptr;
12178         ctf_calc_rwin(so, tp);
12179
12180         /* Reset receive buffer auto scaling when not in bulk receive mode. */
12181         if ((thflags & TH_RST) ||
12182             (tp->t_fin_is_rst && (thflags & TH_FIN)))
12183                 return (__ctf_process_rst(m, th, so, tp,
12184                                           &rack->r_ctl.challenge_ack_ts,
12185                                           &rack->r_ctl.challenge_ack_cnt));
12186         /*
12187          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
12188          * synchronized state.
12189          */
12190         if (thflags & TH_SYN) {
12191                 ctf_challenge_ack(m, th, tp, &ret_val);
12192                 return (ret_val);
12193         }
12194         /*
12195          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
12196          * it's less than ts_recent, drop it.
12197          */
12198         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
12199             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
12200                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
12201                         return (ret_val);
12202         }
12203         if (_ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val,
12204                               &rack->r_ctl.challenge_ack_ts,
12205                               &rack->r_ctl.challenge_ack_cnt)) {
12206                 return (ret_val);
12207         }
12208         /*
12209          * If new data are received on a connection after the user processes
12210          * are gone, then RST the other end.
12211          */
12212         if ((tp->t_flags & TF_CLOSED) && tlen &&
12213             rack_check_data_after_close(m, tp, &tlen, th, so))
12214                 return (1);
12215         /*
12216          * If last ACK falls within this segment's sequence numbers, record
12217          * its timestamp. NOTE: 1) That the test incorporates suggestions
12218          * from the latest proposal of the tcplw@cray.com list (Braden
12219          * 1993/04/26). 2) That updating only on newer timestamps interferes
12220          * with our earlier PAWS tests, so this check should be solely
12221          * predicated on the sequence space of this segment. 3) That we
12222          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
12223          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
12224          * SEG.Len, This modified check allows us to overcome RFC1323's
12225          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
12226          * p.869. In such cases, we can still calculate the RTT correctly
12227          * when RCV.NXT == Last.ACK.Sent.
12228          */
12229         if ((to->to_flags & TOF_TS) != 0 &&
12230             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
12231             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
12232             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
12233                 tp->ts_recent_age = tcp_ts_getticks();
12234                 tp->ts_recent = to->to_tsval;
12235         }
12236         /*
12237          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
12238          * is on (half-synchronized state), then queue data for later
12239          * processing; else drop segment and return.
12240          */
12241         if ((thflags & TH_ACK) == 0) {
12242                 if (tp->t_flags & TF_NEEDSYN) {
12243                         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
12244                             tiwin, thflags, nxt_pkt));
12245                 } else if (tp->t_flags & TF_ACKNOW) {
12246                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
12247                         ((struct tcp_rack *)tp->t_fb_ptr)->r_wanted_output = 1;
12248                         return (ret_val);
12249                 } else {
12250                         ctf_do_drop(m, NULL);
12251                         return (0);
12252                 }
12253         }
12254         /*
12255          * Ack processing.
12256          */
12257         if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
12258                 return (ret_val);
12259         }
12260         if (sbavail(&so->so_snd)) {
12261                 if (ctf_progress_timeout_check(tp, true)) {
12262                         rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
12263                                                 tp, tick, PROGRESS_DROP, __LINE__);
12264                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
12265                         return (1);
12266                 }
12267         }
12268         return (rack_process_data(m, th, so, tp, drop_hdrlen, tlen,
12269             tiwin, thflags, nxt_pkt));
12270 }
12271
12272 static void inline
12273 rack_clear_rate_sample(struct tcp_rack *rack)
12274 {
12275         rack->r_ctl.rack_rs.rs_flags = RACK_RTT_EMPTY;
12276         rack->r_ctl.rack_rs.rs_rtt_cnt = 0;
12277         rack->r_ctl.rack_rs.rs_rtt_tot = 0;
12278 }
12279
12280 static void
12281 rack_set_pace_segments(struct tcpcb *tp, struct tcp_rack *rack, uint32_t line, uint64_t *fill_override)
12282 {
12283         uint64_t bw_est, rate_wanted;
12284         int chged = 0;
12285         uint32_t user_max, orig_min, orig_max;
12286
12287         orig_min = rack->r_ctl.rc_pace_min_segs;
12288         orig_max = rack->r_ctl.rc_pace_max_segs;
12289         user_max = ctf_fixed_maxseg(tp) * rack->rc_user_set_max_segs;
12290         if (ctf_fixed_maxseg(tp) != rack->r_ctl.rc_pace_min_segs)
12291                 chged = 1;
12292         rack->r_ctl.rc_pace_min_segs = ctf_fixed_maxseg(tp);
12293         if (rack->use_fixed_rate || rack->rc_force_max_seg) {
12294                 if (user_max != rack->r_ctl.rc_pace_max_segs)
12295                         chged = 1;
12296         }
12297         if (rack->rc_force_max_seg) {
12298                 rack->r_ctl.rc_pace_max_segs = user_max;
12299         } else if (rack->use_fixed_rate) {
12300                 bw_est = rack_get_bw(rack);
12301                 if ((rack->r_ctl.crte == NULL) ||
12302                     (bw_est != rack->r_ctl.crte->rate)) {
12303                         rack->r_ctl.rc_pace_max_segs = user_max;
12304                 } else {
12305                         /* We are pacing right at the hardware rate */
12306                         uint32_t segsiz;
12307
12308                         segsiz = min(ctf_fixed_maxseg(tp),
12309                                      rack->r_ctl.rc_pace_min_segs);
12310                         rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size(
12311                                                            tp, bw_est, segsiz, 0,
12312                                                            rack->r_ctl.crte, NULL);
12313                 }
12314         } else if (rack->rc_always_pace) {
12315                 if (rack->r_ctl.gp_bw ||
12316 #ifdef NETFLIX_PEAKRATE
12317                     rack->rc_tp->t_maxpeakrate ||
12318 #endif
12319                     rack->r_ctl.init_rate) {
12320                         /* We have a rate of some sort set */
12321                         uint32_t  orig;
12322
12323                         bw_est = rack_get_bw(rack);
12324                         orig = rack->r_ctl.rc_pace_max_segs;
12325                         if (fill_override)
12326                                 rate_wanted = *fill_override;
12327                         else
12328                                 rate_wanted = rack_get_output_bw(rack, bw_est, NULL, NULL);
12329                         if (rate_wanted) {
12330                                 /* We have something */
12331                                 rack->r_ctl.rc_pace_max_segs = rack_get_pacing_len(rack,
12332                                                                                    rate_wanted,
12333                                                                                    ctf_fixed_maxseg(rack->rc_tp));
12334                         } else
12335                                 rack->r_ctl.rc_pace_max_segs = rack->r_ctl.rc_pace_min_segs;
12336                         if (orig != rack->r_ctl.rc_pace_max_segs)
12337                                 chged = 1;
12338                 } else if ((rack->r_ctl.gp_bw == 0) &&
12339                            (rack->r_ctl.rc_pace_max_segs == 0)) {
12340                         /*
12341                          * If we have nothing limit us to bursting
12342                          * out IW sized pieces.
12343                          */
12344                         chged = 1;
12345                         rack->r_ctl.rc_pace_max_segs = rc_init_window(rack);
12346                 }
12347         }
12348         if (rack->r_ctl.rc_pace_max_segs > PACE_MAX_IP_BYTES) {
12349                 chged = 1;
12350                 rack->r_ctl.rc_pace_max_segs = PACE_MAX_IP_BYTES;
12351         }
12352         if (chged)
12353                 rack_log_type_pacing_sizes(tp, rack, orig_min, orig_max, line, 2);
12354 }
12355
12356
12357 static void
12358 rack_init_fsb_block(struct tcpcb *tp, struct tcp_rack *rack)
12359 {
12360 #ifdef INET6
12361         struct ip6_hdr *ip6 = NULL;
12362 #endif
12363 #ifdef INET
12364         struct ip *ip = NULL;
12365 #endif
12366         struct udphdr *udp = NULL;
12367
12368         /* Ok lets fill in the fast block, it can only be used with no IP options! */
12369 #ifdef INET6
12370         if (rack->r_is_v6) {
12371                 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
12372                 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr;
12373                 if (tp->t_port) {
12374                         rack->r_ctl.fsb.tcp_ip_hdr_len += sizeof(struct udphdr);
12375                         udp = (struct udphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
12376                         udp->uh_sport = htons(V_tcp_udp_tunneling_port);
12377                         udp->uh_dport = tp->t_port;
12378                         rack->r_ctl.fsb.udp = udp;
12379                         rack->r_ctl.fsb.th = (struct tcphdr *)(udp + 1);
12380                 } else
12381                 {
12382                         rack->r_ctl.fsb.th = (struct tcphdr *)(ip6 + 1);
12383                         rack->r_ctl.fsb.udp = NULL;
12384                 }
12385                 tcpip_fillheaders(rack->rc_inp,
12386                                   tp->t_port,
12387                                   ip6, rack->r_ctl.fsb.th);
12388         } else
12389 #endif                          /* INET6 */
12390         {
12391                 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct tcpiphdr);
12392                 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
12393                 if (tp->t_port) {
12394                         rack->r_ctl.fsb.tcp_ip_hdr_len += sizeof(struct udphdr);
12395                         udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip));
12396                         udp->uh_sport = htons(V_tcp_udp_tunneling_port);
12397                         udp->uh_dport = tp->t_port;
12398                         rack->r_ctl.fsb.udp = udp;
12399                         rack->r_ctl.fsb.th = (struct tcphdr *)(udp + 1);
12400                 } else
12401                 {
12402                         rack->r_ctl.fsb.udp = NULL;
12403                         rack->r_ctl.fsb.th = (struct tcphdr *)(ip + 1);
12404                 }
12405                 tcpip_fillheaders(rack->rc_inp,
12406                                   tp->t_port,
12407                                   ip, rack->r_ctl.fsb.th);
12408         }
12409         rack->r_fsb_inited = 1;
12410 }
12411
12412 static int
12413 rack_init_fsb(struct tcpcb *tp, struct tcp_rack *rack)
12414 {
12415         /*
12416          * Allocate the larger of spaces V6 if available else just
12417          * V4 and include udphdr (overbook)
12418          */
12419 #ifdef INET6
12420         rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct ip6_hdr) + sizeof(struct tcphdr) + sizeof(struct udphdr);
12421 #else
12422         rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct tcpiphdr) + sizeof(struct udphdr);
12423 #endif
12424         rack->r_ctl.fsb.tcp_ip_hdr = malloc(rack->r_ctl.fsb.tcp_ip_hdr_len,
12425                                             M_TCPFSB, M_NOWAIT|M_ZERO);
12426         if (rack->r_ctl.fsb.tcp_ip_hdr == NULL) {
12427                 return (ENOMEM);
12428         }
12429         rack->r_fsb_inited = 0;
12430         return (0);
12431 }
12432
12433 static int
12434 rack_init(struct tcpcb *tp)
12435 {
12436         struct tcp_rack *rack = NULL;
12437 #ifdef INVARIANTS
12438         struct rack_sendmap *insret;
12439 #endif
12440         uint32_t iwin, snt, us_cts;
12441         int err;
12442
12443         tp->t_fb_ptr = uma_zalloc(rack_pcb_zone, M_NOWAIT);
12444         if (tp->t_fb_ptr == NULL) {
12445                 /*
12446                  * We need to allocate memory but cant. The INP and INP_INFO
12447                  * locks and they are recursive (happens during setup. So a
12448                  * scheme to drop the locks fails :(
12449                  *
12450                  */
12451                 return (ENOMEM);
12452         }
12453         memset(tp->t_fb_ptr, 0, sizeof(struct tcp_rack));
12454
12455         rack = (struct tcp_rack *)tp->t_fb_ptr;
12456         RB_INIT(&rack->r_ctl.rc_mtree);
12457         TAILQ_INIT(&rack->r_ctl.rc_free);
12458         TAILQ_INIT(&rack->r_ctl.rc_tmap);
12459         rack->rc_tp = tp;
12460         rack->rc_inp = tp->t_inpcb;
12461         /* Set the flag */
12462         rack->r_is_v6 = (tp->t_inpcb->inp_vflag & INP_IPV6) != 0;
12463         /* Probably not needed but lets be sure */
12464         rack_clear_rate_sample(rack);
12465         /*
12466          * Save off the default values, socket options will poke
12467          * at these if pacing is not on or we have not yet
12468          * reached where pacing is on (gp_ready/fixed enabled).
12469          * When they get set into the CC module (when gp_ready
12470          * is enabled or we enable fixed) then we will set these
12471          * values into the CC and place in here the old values
12472          * so we have a restoral. Then we will set the flag
12473          * rc_pacing_cc_set. That way whenever we turn off pacing
12474          * or switch off this stack, we will know to go restore
12475          * the saved values.
12476          */
12477         rack->r_ctl.rc_saved_beta.beta = V_newreno_beta_ecn;
12478         rack->r_ctl.rc_saved_beta.beta_ecn = V_newreno_beta_ecn;
12479         /* We want abe like behavior as well */
12480         rack->r_ctl.rc_saved_beta.newreno_flags |= CC_NEWRENO_BETA_ECN_ENABLED;
12481         rack->r_ctl.rc_reorder_fade = rack_reorder_fade;
12482         rack->rc_allow_data_af_clo = rack_ignore_data_after_close;
12483         rack->r_ctl.rc_tlp_threshold = rack_tlp_thresh;
12484         rack->r_ctl.roundends = tp->snd_max;
12485         if (use_rack_rr)
12486                 rack->use_rack_rr = 1;
12487         if (V_tcp_delack_enabled)
12488                 tp->t_delayed_ack = 1;
12489         else
12490                 tp->t_delayed_ack = 0;
12491 #ifdef TCP_ACCOUNTING
12492         if (rack_tcp_accounting) {
12493                 tp->t_flags2 |= TF2_TCP_ACCOUNTING;
12494         }
12495 #endif
12496         if (rack_enable_shared_cwnd)
12497                 rack->rack_enable_scwnd = 1;
12498         rack->rc_user_set_max_segs = rack_hptsi_segments;
12499         rack->rc_force_max_seg = 0;
12500         if (rack_use_imac_dack)
12501                 rack->rc_dack_mode = 1;
12502         TAILQ_INIT(&rack->r_ctl.opt_list);
12503         rack->r_ctl.rc_reorder_shift = rack_reorder_thresh;
12504         rack->r_ctl.rc_pkt_delay = rack_pkt_delay;
12505         rack->r_ctl.rc_tlp_cwnd_reduce = rack_lower_cwnd_at_tlp;
12506         rack->r_ctl.rc_lowest_us_rtt = 0xffffffff;
12507         rack->r_ctl.rc_highest_us_rtt = 0;
12508         rack->r_ctl.bw_rate_cap = rack_bw_rate_cap;
12509         rack->r_ctl.timer_slop = TICKS_2_USEC(tcp_rexmit_slop);
12510         if (rack_use_cmp_acks)
12511                 rack->r_use_cmp_ack = 1;
12512         if (rack_disable_prr)
12513                 rack->rack_no_prr = 1;
12514         if (rack_gp_no_rec_chg)
12515                 rack->rc_gp_no_rec_chg = 1;
12516         if (rack_pace_every_seg && tcp_can_enable_pacing()) {
12517                 rack->rc_always_pace = 1;
12518                 if (rack->use_fixed_rate || rack->gp_ready)
12519                         rack_set_cc_pacing(rack);
12520         } else
12521                 rack->rc_always_pace = 0;
12522         if (rack_enable_mqueue_for_nonpaced || rack->r_use_cmp_ack)
12523                 rack->r_mbuf_queue = 1;
12524         else
12525                 rack->r_mbuf_queue = 0;
12526         if  (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
12527                 tp->t_inpcb->inp_flags2 |= INP_SUPPORTS_MBUFQ;
12528         else
12529                 tp->t_inpcb->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
12530         rack_set_pace_segments(tp, rack, __LINE__, NULL);
12531         if (rack_limits_scwnd)
12532                 rack->r_limit_scw = 1;
12533         else
12534                 rack->r_limit_scw = 0;
12535         rack->rc_labc = V_tcp_abc_l_var;
12536         rack->r_ctl.rc_high_rwnd = tp->snd_wnd;
12537         rack->r_ctl.cwnd_to_use = tp->snd_cwnd;
12538         rack->r_ctl.rc_rate_sample_method = rack_rate_sample_method;
12539         rack->rack_tlp_threshold_use = rack_tlp_threshold_use;
12540         rack->r_ctl.rc_prr_sendalot = rack_send_a_lot_in_prr;
12541         rack->r_ctl.rc_min_to = rack_min_to;
12542         microuptime(&rack->r_ctl.act_rcv_time);
12543         rack->r_ctl.rc_last_time_decay = rack->r_ctl.act_rcv_time;
12544         rack->rc_init_win = rack_default_init_window;
12545         rack->r_ctl.rack_per_of_gp_ss = rack_per_of_gp_ss;
12546         if (rack_hw_up_only)
12547                 rack->r_up_only = 1;
12548         if (rack_do_dyn_mul) {
12549                 /* When dynamic adjustment is on CA needs to start at 100% */
12550                 rack->rc_gp_dyn_mul = 1;
12551                 if (rack_do_dyn_mul >= 100)
12552                         rack->r_ctl.rack_per_of_gp_ca = rack_do_dyn_mul;
12553         } else
12554                 rack->r_ctl.rack_per_of_gp_ca = rack_per_of_gp_ca;
12555         rack->r_ctl.rack_per_of_gp_rec = rack_per_of_gp_rec;
12556         rack->r_ctl.rack_per_of_gp_probertt = rack_per_of_gp_probertt;
12557         rack->r_ctl.rc_tlp_rxt_last_time = tcp_tv_to_mssectick(&rack->r_ctl.act_rcv_time);
12558         setup_time_filter_small(&rack->r_ctl.rc_gp_min_rtt, FILTER_TYPE_MIN,
12559                                 rack_probertt_filter_life);
12560         us_cts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time);
12561         rack->r_ctl.rc_lower_rtt_us_cts = us_cts;
12562         rack->r_ctl.rc_time_of_last_probertt = us_cts;
12563         rack->r_ctl.challenge_ack_ts = tcp_ts_getticks();
12564         rack->r_ctl.rc_time_probertt_starts = 0;
12565         if (rack_dsack_std_based & 0x1) {
12566                 /* Basically this means all rack timers are at least (srtt + 1/4 srtt) */
12567                 rack->rc_rack_tmr_std_based = 1;
12568         }
12569         if (rack_dsack_std_based & 0x2) {
12570                 /* Basically this means  rack timers are extended based on dsack by up to (2 * srtt) */
12571                 rack->rc_rack_use_dsack = 1;
12572         }
12573         /* We require at least one measurement, even if the sysctl is 0 */
12574         if (rack_req_measurements)
12575                 rack->r_ctl.req_measurements = rack_req_measurements;
12576         else
12577                 rack->r_ctl.req_measurements = 1;
12578         if (rack_enable_hw_pacing)
12579                 rack->rack_hdw_pace_ena = 1;
12580         if (rack_hw_rate_caps)
12581                 rack->r_rack_hw_rate_caps = 1;
12582         /* Do we force on detection? */
12583 #ifdef NETFLIX_EXP_DETECTION
12584         if (tcp_force_detection)
12585                 rack->do_detection = 1;
12586         else
12587 #endif
12588                 rack->do_detection = 0;
12589         if (rack_non_rxt_use_cr)
12590                 rack->rack_rec_nonrxt_use_cr = 1;
12591         err = rack_init_fsb(tp, rack);
12592         if (err) {
12593                 uma_zfree(rack_pcb_zone, tp->t_fb_ptr);
12594                 tp->t_fb_ptr = NULL;
12595                 return (err);
12596         }
12597         if (tp->snd_una != tp->snd_max) {
12598                 /* Create a send map for the current outstanding data */
12599                 struct rack_sendmap *rsm;
12600
12601                 rsm = rack_alloc(rack);
12602                 if (rsm == NULL) {
12603                         uma_zfree(rack_pcb_zone, tp->t_fb_ptr);
12604                         tp->t_fb_ptr = NULL;
12605                         return (ENOMEM);
12606                 }
12607                 rsm->r_no_rtt_allowed = 1;
12608                 rsm->r_tim_lastsent[0] = rack_to_usec_ts(&rack->r_ctl.act_rcv_time);
12609                 rsm->r_rtr_cnt = 1;
12610                 rsm->r_rtr_bytes = 0;
12611                 if (tp->t_flags & TF_SENTFIN)
12612                         rsm->r_flags |= RACK_HAS_FIN;
12613                 rsm->r_end = tp->snd_max;
12614                 if (tp->snd_una == tp->iss) {
12615                         /* The data space is one beyond snd_una */
12616                         rsm->r_flags |= RACK_HAS_SYN;
12617                         rsm->r_start = tp->iss;
12618                         rsm->r_end = rsm->r_start + (tp->snd_max - tp->snd_una);
12619                 } else
12620                         rsm->r_start = tp->snd_una;
12621                 rsm->r_dupack = 0;
12622                 if (rack->rc_inp->inp_socket->so_snd.sb_mb != NULL) {
12623                         rsm->m = sbsndmbuf(&rack->rc_inp->inp_socket->so_snd, 0, &rsm->soff);
12624                         if (rsm->m)
12625                                 rsm->orig_m_len = rsm->m->m_len;
12626                         else
12627                                 rsm->orig_m_len = 0;
12628                 } else {
12629                         /*
12630                          * This can happen if we have a stand-alone FIN or
12631                          *  SYN.
12632                          */
12633                         rsm->m = NULL;
12634                         rsm->orig_m_len = 0;
12635                         rsm->soff = 0;
12636                 }
12637 #ifndef INVARIANTS
12638                 (void)RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
12639 #else
12640                 insret = RB_INSERT(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
12641                 if (insret != NULL) {
12642                         panic("Insert in rb tree fails ret:%p rack:%p rsm:%p",
12643                               insret, rack, rsm);
12644                 }
12645 #endif
12646                 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_tmap, rsm, r_tnext);
12647                 rsm->r_in_tmap = 1;
12648         }
12649         /*
12650          * Timers in Rack are kept in microseconds so lets
12651          * convert any initial incoming variables
12652          * from ticks into usecs. Note that we
12653          * also change the values of t_srtt and t_rttvar, if
12654          * they are non-zero. They are kept with a 5
12655          * bit decimal so we have to carefully convert
12656          * these to get the full precision.
12657          */
12658         rack_convert_rtts(tp);
12659         tp->t_rttlow = TICKS_2_USEC(tp->t_rttlow);
12660         if (rack_do_hystart) {
12661                 tp->ccv->flags |= CCF_HYSTART_ALLOWED;
12662                 if (rack_do_hystart > 1)
12663                         tp->ccv->flags |= CCF_HYSTART_CAN_SH_CWND;
12664                 if (rack_do_hystart > 2)
12665                         tp->ccv->flags |= CCF_HYSTART_CONS_SSTH;
12666         }
12667         if (rack_def_profile)
12668                 rack_set_profile(rack, rack_def_profile);
12669         /* Cancel the GP measurement in progress */
12670         tp->t_flags &= ~TF_GPUTINPROG;
12671         if (SEQ_GT(tp->snd_max, tp->iss))
12672                 snt = tp->snd_max - tp->iss;
12673         else
12674                 snt = 0;
12675         iwin = rc_init_window(rack);
12676         if (snt < iwin) {
12677                 /* We are not past the initial window
12678                  * so we need to make sure cwnd is
12679                  * correct.
12680                  */
12681                 if (tp->snd_cwnd < iwin)
12682                         tp->snd_cwnd = iwin;
12683                 /*
12684                  * If we are within the initial window
12685                  * we want ssthresh to be unlimited. Setting
12686                  * it to the rwnd (which the default stack does
12687                  * and older racks) is not really a good idea
12688                  * since we want to be in SS and grow both the
12689                  * cwnd and the rwnd (via dynamic rwnd growth). If
12690                  * we set it to the rwnd then as the peer grows its
12691                  * rwnd we will be stuck in CA and never hit SS.
12692                  *
12693                  * Its far better to raise it up high (this takes the
12694                  * risk that there as been a loss already, probably
12695                  * we should have an indicator in all stacks of loss
12696                  * but we don't), but considering the normal use this
12697                  * is a risk worth taking. The consequences of not
12698                  * hitting SS are far worse than going one more time
12699                  * into it early on (before we have sent even a IW).
12700                  * It is highly unlikely that we will have had a loss
12701                  * before getting the IW out.
12702                  */
12703                 tp->snd_ssthresh = 0xffffffff;
12704         }
12705         rack_stop_all_timers(tp);
12706         /* Lets setup the fsb block */
12707         rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0);
12708         rack_log_rtt_shrinks(rack,  us_cts,  tp->t_rxtcur,
12709                              __LINE__, RACK_RTTS_INIT);
12710         return (0);
12711 }
12712
12713 static int
12714 rack_handoff_ok(struct tcpcb *tp)
12715 {
12716         if ((tp->t_state == TCPS_CLOSED) ||
12717             (tp->t_state == TCPS_LISTEN)) {
12718                 /* Sure no problem though it may not stick */
12719                 return (0);
12720         }
12721         if ((tp->t_state == TCPS_SYN_SENT) ||
12722             (tp->t_state == TCPS_SYN_RECEIVED)) {
12723                 /*
12724                  * We really don't know if you support sack,
12725                  * you have to get to ESTAB or beyond to tell.
12726                  */
12727                 return (EAGAIN);
12728         }
12729         if ((tp->t_flags & TF_SENTFIN) && ((tp->snd_max - tp->snd_una) > 1)) {
12730                 /*
12731                  * Rack will only send a FIN after all data is acknowledged.
12732                  * So in this case we have more data outstanding. We can't
12733                  * switch stacks until either all data and only the FIN
12734                  * is left (in which case rack_init() now knows how
12735                  * to deal with that) <or> all is acknowledged and we
12736                  * are only left with incoming data, though why you
12737                  * would want to switch to rack after all data is acknowledged
12738                  * I have no idea (rrs)!
12739                  */
12740                 return (EAGAIN);
12741         }
12742         if ((tp->t_flags & TF_SACK_PERMIT) || rack_sack_not_required){
12743                 return (0);
12744         }
12745         /*
12746          * If we reach here we don't do SACK on this connection so we can
12747          * never do rack.
12748          */
12749         return (EINVAL);
12750 }
12751
12752
12753 static void
12754 rack_fini(struct tcpcb *tp, int32_t tcb_is_purged)
12755 {
12756         if (tp->t_fb_ptr) {
12757                 struct tcp_rack *rack;
12758                 struct rack_sendmap *rsm, *nrsm;
12759 #ifdef INVARIANTS
12760                 struct rack_sendmap *rm;
12761 #endif
12762
12763                 rack = (struct tcp_rack *)tp->t_fb_ptr;
12764                 if (tp->t_in_pkt) {
12765                         /*
12766                          * It is unsafe to process the packets since a
12767                          * reset may be lurking in them (its rare but it
12768                          * can occur). If we were to find a RST, then we
12769                          * would end up dropping the connection and the
12770                          * INP lock, so when we return the caller (tcp_usrreq)
12771                          * will blow up when it trys to unlock the inp.
12772                          */
12773                         struct mbuf *save, *m;
12774
12775                         m = tp->t_in_pkt;
12776                         tp->t_in_pkt = NULL;
12777                         tp->t_tail_pkt = NULL;
12778                         while (m) {
12779                                 save = m->m_nextpkt;
12780                                 m->m_nextpkt = NULL;
12781                                 m_freem(m);
12782                                 m = save;
12783                         }
12784                 }
12785                 tp->t_flags &= ~TF_FORCEDATA;
12786 #ifdef NETFLIX_SHARED_CWND
12787                 if (rack->r_ctl.rc_scw) {
12788                         uint32_t limit;
12789
12790                         if (rack->r_limit_scw)
12791                                 limit = max(1, rack->r_ctl.rc_lowest_us_rtt);
12792                         else
12793                                 limit = 0;
12794                         tcp_shared_cwnd_free_full(tp, rack->r_ctl.rc_scw,
12795                                                   rack->r_ctl.rc_scw_index,
12796                                                   limit);
12797                         rack->r_ctl.rc_scw = NULL;
12798                 }
12799 #endif
12800                 if (rack->r_ctl.fsb.tcp_ip_hdr) {
12801                         free(rack->r_ctl.fsb.tcp_ip_hdr, M_TCPFSB);
12802                         rack->r_ctl.fsb.tcp_ip_hdr = NULL;
12803                         rack->r_ctl.fsb.th = NULL;
12804                 }
12805                 /* Convert back to ticks, with  */
12806                 if (tp->t_srtt > 1) {
12807                         uint32_t val, frac;
12808
12809                         val = USEC_2_TICKS(tp->t_srtt);
12810                         frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz);
12811                         tp->t_srtt = val << TCP_RTT_SHIFT;
12812                         /*
12813                          * frac is the fractional part here is left
12814                          * over from converting to hz and shifting.
12815                          * We need to convert this to the 5 bit
12816                          * remainder.
12817                          */
12818                         if (frac) {
12819                                 if (hz == 1000) {
12820                                         frac = (((uint64_t)frac *  (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
12821                                 } else {
12822                                         frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
12823                                 }
12824                                 tp->t_srtt += frac;
12825                         }
12826                 }
12827                 if (tp->t_rttvar) {
12828                         uint32_t val, frac;
12829
12830                         val = USEC_2_TICKS(tp->t_rttvar);
12831                         frac = tp->t_srtt % (HPTS_USEC_IN_SEC / hz);
12832                         tp->t_rttvar = val <<  TCP_RTTVAR_SHIFT;
12833                         /*
12834                          * frac is the fractional part here is left
12835                          * over from converting to hz and shifting.
12836                          * We need to convert this to the 5 bit
12837                          * remainder.
12838                          */
12839                         if (frac) {
12840                                 if (hz == 1000) {
12841                                         frac = (((uint64_t)frac *  (uint64_t)TCP_RTT_SCALE) / (uint64_t)HPTS_USEC_IN_MSEC);
12842                                 } else {
12843                                         frac = (((uint64_t)frac * (uint64_t)(hz) * (uint64_t)TCP_RTT_SCALE) /(uint64_t)HPTS_USEC_IN_SEC);
12844                                 }
12845                                 tp->t_rttvar += frac;
12846                         }
12847                 }
12848                 tp->t_rxtcur = USEC_2_TICKS(tp->t_rxtcur);
12849                 tp->t_rttlow = USEC_2_TICKS(tp->t_rttlow);
12850                 if (rack->rc_always_pace) {
12851                         tcp_decrement_paced_conn();
12852                         rack_undo_cc_pacing(rack);
12853                         rack->rc_always_pace = 0;
12854                 }
12855                 /* Clean up any options if they were not applied */
12856                 while (!TAILQ_EMPTY(&rack->r_ctl.opt_list)) {
12857                         struct deferred_opt_list *dol;
12858
12859                         dol = TAILQ_FIRST(&rack->r_ctl.opt_list);
12860                         TAILQ_REMOVE(&rack->r_ctl.opt_list, dol, next);
12861                         free(dol, M_TCPDO);
12862                 }
12863                 /* rack does not use force data but other stacks may clear it */
12864                 if (rack->r_ctl.crte != NULL) {
12865                         tcp_rel_pacing_rate(rack->r_ctl.crte, tp);
12866                         rack->rack_hdrw_pacing = 0;
12867                         rack->r_ctl.crte = NULL;
12868                 }
12869 #ifdef TCP_BLACKBOX
12870                 tcp_log_flowend(tp);
12871 #endif
12872                 RB_FOREACH_SAFE(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree, nrsm) {
12873 #ifndef INVARIANTS
12874                         (void)RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
12875 #else
12876                         rm = RB_REMOVE(rack_rb_tree_head, &rack->r_ctl.rc_mtree, rsm);
12877                         if (rm != rsm) {
12878                                 panic("At fini, rack:%p rsm:%p rm:%p",
12879                                       rack, rsm, rm);
12880                         }
12881 #endif
12882                         uma_zfree(rack_zone, rsm);
12883                 }
12884                 rsm = TAILQ_FIRST(&rack->r_ctl.rc_free);
12885                 while (rsm) {
12886                         TAILQ_REMOVE(&rack->r_ctl.rc_free, rsm, r_tnext);
12887                         uma_zfree(rack_zone, rsm);
12888                         rsm = TAILQ_FIRST(&rack->r_ctl.rc_free);
12889                 }
12890                 rack->rc_free_cnt = 0;
12891                 uma_zfree(rack_pcb_zone, tp->t_fb_ptr);
12892                 tp->t_fb_ptr = NULL;
12893         }
12894         if (tp->t_inpcb) {
12895                 tp->t_inpcb->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
12896                 tp->t_inpcb->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
12897                 tp->t_inpcb->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
12898                 tp->t_inpcb->inp_flags2 &= ~INP_MBUF_ACKCMP;
12899                 /* Cancel the GP measurement in progress */
12900                 tp->t_flags &= ~TF_GPUTINPROG;
12901                 tp->t_inpcb->inp_flags2 &= ~INP_MBUF_L_ACKS;
12902         }
12903         /* Make sure snd_nxt is correctly set */
12904         tp->snd_nxt = tp->snd_max;
12905 }
12906
12907 static void
12908 rack_set_state(struct tcpcb *tp, struct tcp_rack *rack)
12909 {
12910         if ((rack->r_state == TCPS_CLOSED) && (tp->t_state != TCPS_CLOSED)) {
12911                 rack->r_is_v6 = (tp->t_inpcb->inp_vflag & INP_IPV6) != 0;
12912         }
12913         switch (tp->t_state) {
12914         case TCPS_SYN_SENT:
12915                 rack->r_state = TCPS_SYN_SENT;
12916                 rack->r_substate = rack_do_syn_sent;
12917                 break;
12918         case TCPS_SYN_RECEIVED:
12919                 rack->r_state = TCPS_SYN_RECEIVED;
12920                 rack->r_substate = rack_do_syn_recv;
12921                 break;
12922         case TCPS_ESTABLISHED:
12923                 rack_set_pace_segments(tp, rack, __LINE__, NULL);
12924                 rack->r_state = TCPS_ESTABLISHED;
12925                 rack->r_substate = rack_do_established;
12926                 break;
12927         case TCPS_CLOSE_WAIT:
12928                 rack_set_pace_segments(tp, rack, __LINE__, NULL);
12929                 rack->r_state = TCPS_CLOSE_WAIT;
12930                 rack->r_substate = rack_do_close_wait;
12931                 break;
12932         case TCPS_FIN_WAIT_1:
12933                 rack_set_pace_segments(tp, rack, __LINE__, NULL);
12934                 rack->r_state = TCPS_FIN_WAIT_1;
12935                 rack->r_substate = rack_do_fin_wait_1;
12936                 break;
12937         case TCPS_CLOSING:
12938                 rack_set_pace_segments(tp, rack, __LINE__, NULL);
12939                 rack->r_state = TCPS_CLOSING;
12940                 rack->r_substate = rack_do_closing;
12941                 break;
12942         case TCPS_LAST_ACK:
12943                 rack_set_pace_segments(tp, rack, __LINE__, NULL);
12944                 rack->r_state = TCPS_LAST_ACK;
12945                 rack->r_substate = rack_do_lastack;
12946                 break;
12947         case TCPS_FIN_WAIT_2:
12948                 rack_set_pace_segments(tp, rack, __LINE__, NULL);
12949                 rack->r_state = TCPS_FIN_WAIT_2;
12950                 rack->r_substate = rack_do_fin_wait_2;
12951                 break;
12952         case TCPS_LISTEN:
12953         case TCPS_CLOSED:
12954         case TCPS_TIME_WAIT:
12955         default:
12956                 break;
12957         };
12958         if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
12959                 rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
12960
12961 }
12962
12963 static void
12964 rack_timer_audit(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb)
12965 {
12966         /*
12967          * We received an ack, and then did not
12968          * call send or were bounced out due to the
12969          * hpts was running. Now a timer is up as well, is
12970          * it the right timer?
12971          */
12972         struct rack_sendmap *rsm;
12973         int tmr_up;
12974
12975         tmr_up = rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
12976         if (rack->rc_in_persist && (tmr_up == PACE_TMR_PERSIT))
12977                 return;
12978         rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
12979         if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&
12980             (tmr_up == PACE_TMR_RXT)) {
12981                 /* Should be an RXT */
12982                 return;
12983         }
12984         if (rsm == NULL) {
12985                 /* Nothing outstanding? */
12986                 if (tp->t_flags & TF_DELACK) {
12987                         if (tmr_up == PACE_TMR_DELACK)
12988                                 /* We are supposed to have delayed ack up and we do */
12989                                 return;
12990                 } else if (sbavail(&tp->t_inpcb->inp_socket->so_snd) && (tmr_up == PACE_TMR_RXT)) {
12991                         /*
12992                          * if we hit enobufs then we would expect the possibility
12993                          * of nothing outstanding and the RXT up (and the hptsi timer).
12994                          */
12995                         return;
12996                 } else if (((V_tcp_always_keepalive ||
12997                              rack->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&
12998                             (tp->t_state <= TCPS_CLOSING)) &&
12999                            (tmr_up == PACE_TMR_KEEP) &&
13000                            (tp->snd_max == tp->snd_una)) {
13001                         /* We should have keep alive up and we do */
13002                         return;
13003                 }
13004         }
13005         if (SEQ_GT(tp->snd_max, tp->snd_una) &&
13006                    ((tmr_up == PACE_TMR_TLP) ||
13007                     (tmr_up == PACE_TMR_RACK) ||
13008                     (tmr_up == PACE_TMR_RXT))) {
13009                 /*
13010                  * Either a Rack, TLP or RXT is fine if  we
13011                  * have outstanding data.
13012                  */
13013                 return;
13014         } else if (tmr_up == PACE_TMR_DELACK) {
13015                 /*
13016                  * If the delayed ack was going to go off
13017                  * before the rtx/tlp/rack timer were going to
13018                  * expire, then that would be the timer in control.
13019                  * Note we don't check the time here trusting the
13020                  * code is correct.
13021                  */
13022                 return;
13023         }
13024         /*
13025          * Ok the timer originally started is not what we want now.
13026          * We will force the hpts to be stopped if any, and restart
13027          * with the slot set to what was in the saved slot.
13028          */
13029         if (tcp_in_hpts(rack->rc_inp)) {
13030                 if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
13031                         uint32_t us_cts;
13032
13033                         us_cts = tcp_get_usecs(NULL);
13034                         if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) {
13035                                 rack->r_early = 1;
13036                                 rack->r_ctl.rc_agg_early += (rack->r_ctl.rc_last_output_to - us_cts);
13037                         }
13038                         rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
13039                 }
13040                 tcp_hpts_remove(tp->t_inpcb);
13041         }
13042         rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
13043         rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0);
13044 }
13045
13046
13047 static void
13048 rack_do_win_updates(struct tcpcb *tp, struct tcp_rack *rack, uint32_t tiwin, uint32_t seq, uint32_t ack, uint32_t cts, uint32_t high_seq)
13049 {
13050         if ((SEQ_LT(tp->snd_wl1, seq) ||
13051             (tp->snd_wl1 == seq && (SEQ_LT(tp->snd_wl2, ack) ||
13052             (tp->snd_wl2 == ack && tiwin > tp->snd_wnd))))) {
13053                 /* keep track of pure window updates */
13054                 if ((tp->snd_wl2 == ack) && (tiwin > tp->snd_wnd))
13055                         KMOD_TCPSTAT_INC(tcps_rcvwinupd);
13056                 tp->snd_wnd = tiwin;
13057                 rack_validate_fo_sendwin_up(tp, rack);
13058                 tp->snd_wl1 = seq;
13059                 tp->snd_wl2 = ack;
13060                 if (tp->snd_wnd > tp->max_sndwnd)
13061                         tp->max_sndwnd = tp->snd_wnd;
13062             rack->r_wanted_output = 1;
13063         } else if ((tp->snd_wl2 == ack) && (tiwin < tp->snd_wnd)) {
13064                 tp->snd_wnd = tiwin;
13065                 rack_validate_fo_sendwin_up(tp, rack);
13066                 tp->snd_wl1 = seq;
13067                 tp->snd_wl2 = ack;
13068         } else {
13069                 /* Not a valid win update */
13070                 return;
13071         }
13072         if (tp->snd_wnd > tp->max_sndwnd)
13073                 tp->max_sndwnd = tp->snd_wnd;
13074         if (tp->snd_wnd < (tp->snd_max - high_seq)) {
13075                 /* The peer collapsed the window */
13076                 rack_collapsed_window(rack);
13077         } else if (rack->rc_has_collapsed)
13078                 rack_un_collapse_window(rack);
13079         /* Do we exit persists? */
13080         if ((rack->rc_in_persist != 0) &&
13081             (tp->snd_wnd >= min((rack->r_ctl.rc_high_rwnd/2),
13082                                 rack->r_ctl.rc_pace_min_segs))) {
13083                 rack_exit_persist(tp, rack, cts);
13084         }
13085         /* Do we enter persists? */
13086         if ((rack->rc_in_persist == 0) &&
13087             (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), rack->r_ctl.rc_pace_min_segs)) &&
13088             TCPS_HAVEESTABLISHED(tp->t_state) &&
13089             ((tp->snd_max == tp->snd_una) || rack->rc_has_collapsed) &&
13090             sbavail(&tp->t_inpcb->inp_socket->so_snd) &&
13091             (sbavail(&tp->t_inpcb->inp_socket->so_snd) > tp->snd_wnd)) {
13092                 /*
13093                  * Here the rwnd is less than
13094                  * the pacing size, we are established,
13095                  * nothing is outstanding, and there is
13096                  * data to send. Enter persists.
13097                  */
13098                 rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime);
13099         }
13100 }
13101
13102 static void
13103 rack_log_input_packet(struct tcpcb *tp, struct tcp_rack *rack, struct tcp_ackent *ae, int ackval, uint32_t high_seq)
13104 {
13105
13106         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
13107                 union tcp_log_stackspecific log;
13108                 struct timeval ltv;
13109                 char tcp_hdr_buf[60];
13110                 struct tcphdr *th;
13111                 struct timespec ts;
13112                 uint32_t orig_snd_una;
13113                 uint8_t xx = 0;
13114
13115 #ifdef NETFLIX_HTTP_LOGGING
13116                 struct http_sendfile_track *http_req;
13117
13118                 if (SEQ_GT(ae->ack, tp->snd_una)) {
13119                         http_req = tcp_http_find_req_for_seq(tp, (ae->ack-1));
13120                 } else {
13121                         http_req = tcp_http_find_req_for_seq(tp, ae->ack);
13122                 }
13123 #endif
13124                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
13125                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
13126                 if (rack->rack_no_prr == 0)
13127                         log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
13128                 else
13129                         log.u_bbr.flex1 = 0;
13130                 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
13131                 log.u_bbr.use_lt_bw <<= 1;
13132                 log.u_bbr.use_lt_bw |= rack->r_might_revert;
13133                 log.u_bbr.flex2 = rack->r_ctl.rc_num_maps_alloced;
13134                 log.u_bbr.inflight = ctf_flight_size(tp, rack->r_ctl.rc_sacked);
13135                 log.u_bbr.pkts_out = tp->t_maxseg;
13136                 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags;
13137                 log.u_bbr.flex7 = 1;
13138                 log.u_bbr.lost = ae->flags;
13139                 log.u_bbr.cwnd_gain = ackval;
13140                 log.u_bbr.pacing_gain = 0x2;
13141                 if (ae->flags & TSTMP_HDWR) {
13142                         /* Record the hardware timestamp if present */
13143                         log.u_bbr.flex3 = M_TSTMP;
13144                         ts.tv_sec = ae->timestamp / 1000000000;
13145                         ts.tv_nsec = ae->timestamp % 1000000000;
13146                         ltv.tv_sec = ts.tv_sec;
13147                         ltv.tv_usec = ts.tv_nsec / 1000;
13148                         log.u_bbr.lt_epoch = tcp_tv_to_usectick(&ltv);
13149                 } else if (ae->flags & TSTMP_LRO) {
13150                         /* Record the LRO the arrival timestamp */
13151                         log.u_bbr.flex3 = M_TSTMP_LRO;
13152                         ts.tv_sec = ae->timestamp / 1000000000;
13153                         ts.tv_nsec = ae->timestamp % 1000000000;
13154                         ltv.tv_sec = ts.tv_sec;
13155                         ltv.tv_usec = ts.tv_nsec / 1000;
13156                         log.u_bbr.flex5 = tcp_tv_to_usectick(&ltv);
13157                 }
13158                 log.u_bbr.timeStamp = tcp_get_usecs(&ltv);
13159                 /* Log the rcv time */
13160                 log.u_bbr.delRate = ae->timestamp;
13161 #ifdef NETFLIX_HTTP_LOGGING
13162                 log.u_bbr.applimited = tp->t_http_closed;
13163                 log.u_bbr.applimited <<= 8;
13164                 log.u_bbr.applimited |= tp->t_http_open;
13165                 log.u_bbr.applimited <<= 8;
13166                 log.u_bbr.applimited |= tp->t_http_req;
13167                 if (http_req) {
13168                         /* Copy out any client req info */
13169                         /* seconds */
13170                         log.u_bbr.pkt_epoch = (http_req->localtime / HPTS_USEC_IN_SEC);
13171                         /* useconds */
13172                         log.u_bbr.delivered = (http_req->localtime % HPTS_USEC_IN_SEC);
13173                         log.u_bbr.rttProp = http_req->timestamp;
13174                         log.u_bbr.cur_del_rate = http_req->start;
13175                         if (http_req->flags & TCP_HTTP_TRACK_FLG_OPEN) {
13176                                 log.u_bbr.flex8 |= 1;
13177                         } else {
13178                                 log.u_bbr.flex8 |= 2;
13179                                 log.u_bbr.bw_inuse = http_req->end;
13180                         }
13181                         log.u_bbr.flex6 = http_req->start_seq;
13182                         if (http_req->flags & TCP_HTTP_TRACK_FLG_COMP) {
13183                                 log.u_bbr.flex8 |= 4;
13184                                 log.u_bbr.epoch = http_req->end_seq;
13185                         }
13186                 }
13187 #endif
13188                 memset(tcp_hdr_buf, 0, sizeof(tcp_hdr_buf));
13189                 th = (struct tcphdr *)tcp_hdr_buf;
13190                 th->th_seq = ae->seq;
13191                 th->th_ack = ae->ack;
13192                 th->th_win = ae->win;
13193                 /* Now fill in the ports */
13194                 th->th_sport = tp->t_inpcb->inp_fport;
13195                 th->th_dport = tp->t_inpcb->inp_lport;
13196                 tcp_set_flags(th, ae->flags);
13197                 /* Now do we have a timestamp option? */
13198                 if (ae->flags & HAS_TSTMP) {
13199                         u_char *cp;
13200                         uint32_t val;
13201
13202                         th->th_off = ((sizeof(struct tcphdr) + TCPOLEN_TSTAMP_APPA) >> 2);
13203                         cp = (u_char *)(th + 1);
13204                         *cp = TCPOPT_NOP;
13205                         cp++;
13206                         *cp = TCPOPT_NOP;
13207                         cp++;
13208                         *cp = TCPOPT_TIMESTAMP;
13209                         cp++;
13210                         *cp = TCPOLEN_TIMESTAMP;
13211                         cp++;
13212                         val = htonl(ae->ts_value);
13213                         bcopy((char *)&val,
13214                               (char *)cp, sizeof(uint32_t));
13215                         val = htonl(ae->ts_echo);
13216                         bcopy((char *)&val,
13217                               (char *)(cp + 4), sizeof(uint32_t));
13218                 } else
13219                         th->th_off = (sizeof(struct tcphdr) >> 2);
13220
13221                 /*
13222                  * For sane logging we need to play a little trick.
13223                  * If the ack were fully processed we would have moved
13224                  * snd_una to high_seq, but since compressed acks are
13225                  * processed in two phases, at this point (logging) snd_una
13226                  * won't be advanced. So we would see multiple acks showing
13227                  * the advancement. We can prevent that by "pretending" that
13228                  * snd_una was advanced and then un-advancing it so that the
13229                  * logging code has the right value for tlb_snd_una.
13230                  */
13231                 if (tp->snd_una != high_seq) {
13232                         orig_snd_una = tp->snd_una;
13233                         tp->snd_una = high_seq;
13234                         xx = 1;
13235                 } else
13236                         xx = 0;
13237                 TCP_LOG_EVENTP(tp, th,
13238                                &tp->t_inpcb->inp_socket->so_rcv,
13239                                &tp->t_inpcb->inp_socket->so_snd, TCP_LOG_IN, 0,
13240                                0, &log, true, &ltv);
13241                 if (xx) {
13242                         tp->snd_una = orig_snd_una;
13243                 }
13244         }
13245
13246 }
13247
13248 static void
13249 rack_handle_probe_response(struct tcp_rack *rack, uint32_t tiwin, uint32_t us_cts)
13250 {
13251         uint32_t us_rtt;
13252         /*
13253          * A persist or keep-alive was forced out, update our
13254          * min rtt time. Note now worry about lost responses.
13255          * When a subsequent keep-alive or persist times out
13256          * and forced_ack is still on, then the last probe
13257          * was not responded to. In such cases we have a
13258          * sysctl that controls the behavior. Either we apply
13259          * the rtt but with reduced confidence (0). Or we just
13260          * plain don't apply the rtt estimate. Having data flow
13261          * will clear the probe_not_answered flag i.e. cum-ack
13262          * move forward <or> exiting and reentering persists.
13263          */
13264
13265         rack->forced_ack = 0;
13266         rack->rc_tp->t_rxtshift = 0;
13267         if ((rack->rc_in_persist &&
13268              (tiwin == rack->rc_tp->snd_wnd)) ||
13269             (rack->rc_in_persist == 0)) {
13270                 /*
13271                  * In persists only apply the RTT update if this is
13272                  * a response to our window probe. And that
13273                  * means the rwnd sent must match the current
13274                  * snd_wnd. If it does not, then we got a
13275                  * window update ack instead. For keepalive
13276                  * we allow the answer no matter what the window.
13277                  *
13278                  * Note that if the probe_not_answered is set then
13279                  * the forced_ack_ts is the oldest one i.e. the first
13280                  * probe sent that might have been lost. This assures
13281                  * us that if we do calculate an RTT it is longer not
13282                  * some short thing.
13283                  */
13284                 if (rack->rc_in_persist)
13285                         counter_u64_add(rack_persists_acks, 1);
13286                 us_rtt = us_cts - rack->r_ctl.forced_ack_ts;
13287                 if (us_rtt == 0)
13288                         us_rtt = 1;
13289                 if (rack->probe_not_answered == 0) {
13290                         rack_apply_updated_usrtt(rack, us_rtt, us_cts);
13291                         tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 3, NULL, 1);
13292                 } else {
13293                         /* We have a retransmitted probe here too */
13294                         if (rack_apply_rtt_with_reduced_conf) {
13295                                 rack_apply_updated_usrtt(rack, us_rtt, us_cts);
13296                                 tcp_rack_xmit_timer(rack, us_rtt, 0, us_rtt, 0, NULL, 1);
13297                         }
13298                 }
13299         }
13300 }
13301
13302 static int
13303 rack_do_compressed_ack_processing(struct tcpcb *tp, struct socket *so, struct mbuf *m, int nxt_pkt, struct timeval *tv)
13304 {
13305         /*
13306          * Handle a "special" compressed ack mbuf. Each incoming
13307          * ack has only four possible dispositions:
13308          *
13309          * A) It moves the cum-ack forward
13310          * B) It is behind the cum-ack.
13311          * C) It is a window-update ack.
13312          * D) It is a dup-ack.
13313          *
13314          * Note that we can have between 1 -> TCP_COMP_ACK_ENTRIES
13315          * in the incoming mbuf. We also need to still pay attention
13316          * to nxt_pkt since there may be another packet after this
13317          * one.
13318          */
13319 #ifdef TCP_ACCOUNTING
13320         uint64_t ts_val;
13321         uint64_t rdstc;
13322 #endif
13323         int segsiz;
13324         struct timespec ts;
13325         struct tcp_rack *rack;
13326         struct tcp_ackent *ae;
13327         uint32_t tiwin, ms_cts, cts, acked, acked_amount, high_seq, win_seq, the_win, win_upd_ack;
13328         int cnt, i, did_out, ourfinisacked = 0;
13329         struct tcpopt to_holder, *to = NULL;
13330 #ifdef TCP_ACCOUNTING
13331         int win_up_req = 0;
13332 #endif
13333         int nsegs = 0;
13334         int under_pacing = 1;
13335         int recovery = 0;
13336 #ifdef TCP_ACCOUNTING
13337         sched_pin();
13338 #endif
13339         rack = (struct tcp_rack *)tp->t_fb_ptr;
13340         if (rack->gp_ready &&
13341             (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT))
13342                 under_pacing = 0;
13343         else
13344                 under_pacing = 1;
13345
13346         if (rack->r_state != tp->t_state)
13347                 rack_set_state(tp, rack);
13348         if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
13349             (tp->t_flags & TF_GPUTINPROG)) {
13350                 /*
13351                  * We have a goodput in progress
13352                  * and we have entered a late state.
13353                  * Do we have enough data in the sb
13354                  * to handle the GPUT request?
13355                  */
13356                 uint32_t bytes;
13357
13358                 bytes = tp->gput_ack - tp->gput_seq;
13359                 if (SEQ_GT(tp->gput_seq, tp->snd_una))
13360                         bytes += tp->gput_seq - tp->snd_una;
13361                 if (bytes > sbavail(&tp->t_inpcb->inp_socket->so_snd)) {
13362                         /*
13363                          * There are not enough bytes in the socket
13364                          * buffer that have been sent to cover this
13365                          * measurement. Cancel it.
13366                          */
13367                         rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
13368                                                    rack->r_ctl.rc_gp_srtt /*flex1*/,
13369                                                    tp->gput_seq,
13370                                                    0, 0, 18, __LINE__, NULL, 0);
13371                         tp->t_flags &= ~TF_GPUTINPROG;
13372                 }
13373         }
13374         to = &to_holder;
13375         to->to_flags = 0;
13376         KASSERT((m->m_len >= sizeof(struct tcp_ackent)),
13377                 ("tp:%p m_cmpack:%p with invalid len:%u", tp, m, m->m_len));
13378         cnt = m->m_len / sizeof(struct tcp_ackent);
13379         counter_u64_add(rack_multi_single_eq, cnt);
13380         high_seq = tp->snd_una;
13381         the_win = tp->snd_wnd;
13382         win_seq = tp->snd_wl1;
13383         win_upd_ack = tp->snd_wl2;
13384         cts = tcp_tv_to_usectick(tv);
13385         ms_cts = tcp_tv_to_mssectick(tv);
13386         rack->r_ctl.rc_rcvtime = cts;
13387         segsiz = ctf_fixed_maxseg(tp);
13388         if ((rack->rc_gp_dyn_mul) &&
13389             (rack->use_fixed_rate == 0) &&
13390             (rack->rc_always_pace)) {
13391                 /* Check in on probertt */
13392                 rack_check_probe_rtt(rack, cts);
13393         }
13394         for (i = 0; i < cnt; i++) {
13395 #ifdef TCP_ACCOUNTING
13396                 ts_val = get_cyclecount();
13397 #endif
13398                 rack_clear_rate_sample(rack);
13399                 ae = ((mtod(m, struct tcp_ackent *)) + i);
13400                 /* Setup the window */
13401                 tiwin = ae->win << tp->snd_scale;
13402                 if (tiwin > rack->r_ctl.rc_high_rwnd)
13403                         rack->r_ctl.rc_high_rwnd = tiwin;
13404                 /* figure out the type of ack */
13405                 if (SEQ_LT(ae->ack, high_seq)) {
13406                         /* Case B*/
13407                         ae->ack_val_set = ACK_BEHIND;
13408                 } else if (SEQ_GT(ae->ack, high_seq)) {
13409                         /* Case A */
13410                         ae->ack_val_set = ACK_CUMACK;
13411                 } else if ((tiwin == the_win) && (rack->rc_in_persist == 0)){
13412                         /* Case D */
13413                         ae->ack_val_set = ACK_DUPACK;
13414                 } else {
13415                         /* Case C */
13416                         ae->ack_val_set = ACK_RWND;
13417                 }
13418                 rack_log_input_packet(tp, rack, ae, ae->ack_val_set, high_seq);
13419                 /* Validate timestamp */
13420                 if (ae->flags & HAS_TSTMP) {
13421                         /* Setup for a timestamp */
13422                         to->to_flags = TOF_TS;
13423                         ae->ts_echo -= tp->ts_offset;
13424                         to->to_tsecr = ae->ts_echo;
13425                         to->to_tsval = ae->ts_value;
13426                         /*
13427                          * If echoed timestamp is later than the current time, fall back to
13428                          * non RFC1323 RTT calculation.  Normalize timestamp if syncookies
13429                          * were used when this connection was established.
13430                          */
13431                         if (TSTMP_GT(ae->ts_echo, ms_cts))
13432                                 to->to_tsecr = 0;
13433                         if (tp->ts_recent &&
13434                             TSTMP_LT(ae->ts_value, tp->ts_recent)) {
13435                                 if (ctf_ts_check_ac(tp, (ae->flags & 0xff))) {
13436 #ifdef TCP_ACCOUNTING
13437                                         rdstc = get_cyclecount();
13438                                         if (rdstc > ts_val) {
13439                                                 counter_u64_add(tcp_proc_time[ae->ack_val_set] ,
13440                                                                 (rdstc - ts_val));
13441                                                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13442                                                         tp->tcp_proc_time[ae->ack_val_set] += (rdstc - ts_val);
13443                                                 }
13444                                         }
13445 #endif
13446                                         continue;
13447                                 }
13448                         }
13449                         if (SEQ_LEQ(ae->seq, tp->last_ack_sent) &&
13450                             SEQ_LEQ(tp->last_ack_sent, ae->seq)) {
13451                                 tp->ts_recent_age = tcp_ts_getticks();
13452                                 tp->ts_recent = ae->ts_value;
13453                         }
13454                 } else {
13455                         /* Setup for a no options */
13456                         to->to_flags = 0;
13457                 }
13458                 /* Update the rcv time and perform idle reduction possibly */
13459                 if  (tp->t_idle_reduce &&
13460                      (tp->snd_max == tp->snd_una) &&
13461                      (TICKS_2_USEC(ticks - tp->t_rcvtime) >= tp->t_rxtcur)) {
13462                         counter_u64_add(rack_input_idle_reduces, 1);
13463                         rack_cc_after_idle(rack, tp);
13464                 }
13465                 tp->t_rcvtime = ticks;
13466                 /* Now what about ECN? */
13467                 if (tcp_ecn_input_segment(tp, ae->flags, ae->codepoint))
13468                         rack_cong_signal(tp, CC_ECN, ae->ack, __LINE__);
13469 #ifdef TCP_ACCOUNTING
13470                 /* Count for the specific type of ack in */
13471                 counter_u64_add(tcp_cnt_counters[ae->ack_val_set], 1);
13472                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13473                         tp->tcp_cnt_counters[ae->ack_val_set]++;
13474                 }
13475 #endif
13476                 /*
13477                  * Note how we could move up these in the determination
13478                  * above, but we don't so that way the timestamp checks (and ECN)
13479                  * is done first before we do any processing on the ACK.
13480                  * The non-compressed path through the code has this
13481                  * weakness (noted by @jtl) that it actually does some
13482                  * processing before verifying the timestamp information.
13483                  * We don't take that path here which is why we set
13484                  * the ack_val_set first, do the timestamp and ecn
13485                  * processing, and then look at what we have setup.
13486                  */
13487                 if (ae->ack_val_set == ACK_BEHIND) {
13488                         /*
13489                          * Case B flag reordering, if window is not closed
13490                          * or it could be a keep-alive or persists
13491                          */
13492                         if (SEQ_LT(ae->ack, tp->snd_una) && (sbspace(&so->so_rcv) > segsiz)) {
13493                                 rack->r_ctl.rc_reorder_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time);
13494                         }
13495                 } else if (ae->ack_val_set == ACK_DUPACK) {
13496                         /* Case D */
13497                         rack_strike_dupack(rack);
13498                 } else if (ae->ack_val_set == ACK_RWND) {
13499                         /* Case C */
13500                         if ((ae->flags & TSTMP_LRO) || (ae->flags & TSTMP_HDWR)) {
13501                                 ts.tv_sec = ae->timestamp / 1000000000;
13502                                 ts.tv_nsec = ae->timestamp % 1000000000;
13503                                 rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec;
13504                                 rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000;
13505                         } else {
13506                                 rack->r_ctl.act_rcv_time = *tv;
13507                         }
13508                         if (rack->forced_ack) {
13509                                 rack_handle_probe_response(rack, tiwin,
13510                                                            tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time));
13511                         }
13512 #ifdef TCP_ACCOUNTING
13513                         win_up_req = 1;
13514 #endif
13515                         win_upd_ack = ae->ack;
13516                         win_seq = ae->seq;
13517                         the_win = tiwin;
13518                         rack_do_win_updates(tp, rack, the_win, win_seq, win_upd_ack, cts, high_seq);
13519                 } else {
13520                         /* Case A */
13521                         if (SEQ_GT(ae->ack, tp->snd_max)) {
13522                                 /*
13523                                  * We just send an ack since the incoming
13524                                  * ack is beyond the largest seq we sent.
13525                                  */
13526                                 if ((tp->t_flags & TF_ACKNOW) == 0) {
13527                                         ctf_ack_war_checks(tp, &rack->r_ctl.challenge_ack_ts, &rack->r_ctl.challenge_ack_cnt);
13528                                         if (tp->t_flags && TF_ACKNOW)
13529                                                 rack->r_wanted_output = 1;
13530                                 }
13531                         } else {
13532                                 nsegs++;
13533                                 /* If the window changed setup to update */
13534                                 if (tiwin != tp->snd_wnd) {
13535                                         win_upd_ack = ae->ack;
13536                                         win_seq = ae->seq;
13537                                         the_win = tiwin;
13538                                         rack_do_win_updates(tp, rack, the_win, win_seq, win_upd_ack, cts, high_seq);
13539                                 }
13540 #ifdef TCP_ACCOUNTING
13541                                 /* Account for the acks */
13542                                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13543                                         tp->tcp_cnt_counters[CNT_OF_ACKS_IN] += (((ae->ack - high_seq) + segsiz - 1) / segsiz);
13544                                 }
13545                                 counter_u64_add(tcp_cnt_counters[CNT_OF_ACKS_IN],
13546                                                 (((ae->ack - high_seq) + segsiz - 1) / segsiz));
13547 #endif
13548                                 high_seq = ae->ack;
13549                                 if (rack_verbose_logging && (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
13550                                         union tcp_log_stackspecific log;
13551                                         struct timeval tv;
13552
13553                                         memset(&log.u_bbr, 0, sizeof(log.u_bbr));
13554                                         log.u_bbr.timeStamp = tcp_get_usecs(&tv);
13555                                         log.u_bbr.flex1 = high_seq;
13556                                         log.u_bbr.flex2 = rack->r_ctl.roundends;
13557                                         log.u_bbr.flex3 = rack->r_ctl.current_round;
13558                                         log.u_bbr.rttProp = (uint64_t)CC_ALGO(tp)->newround;
13559                                         log.u_bbr.flex8 = 8;
13560                                         tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
13561                                                        0, &log, false, NULL, NULL, 0, &tv);
13562                                 }
13563                                 /*
13564                                  * The draft (v3) calls for us to use SEQ_GEQ, but that
13565                                  * causes issues when we are just going app limited. Lets
13566                                  * instead use SEQ_GT <or> where its equal but more data
13567                                  * is outstanding.
13568                                  */
13569                                 if ((SEQ_GT(high_seq, rack->r_ctl.roundends)) ||
13570                                     ((high_seq == rack->r_ctl.roundends) &&
13571                                      SEQ_GT(tp->snd_max, tp->snd_una))) {
13572                                         rack->r_ctl.current_round++;
13573                                         rack->r_ctl.roundends = tp->snd_max;
13574                                         if (CC_ALGO(tp)->newround != NULL) {
13575                                                 CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round);
13576                                         }
13577                                 }
13578                                 /* Setup our act_rcv_time */
13579                                 if ((ae->flags & TSTMP_LRO) || (ae->flags & TSTMP_HDWR)) {
13580                                         ts.tv_sec = ae->timestamp / 1000000000;
13581                                         ts.tv_nsec = ae->timestamp % 1000000000;
13582                                         rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec;
13583                                         rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000;
13584                                 } else {
13585                                         rack->r_ctl.act_rcv_time = *tv;
13586                                 }
13587                                 rack_process_to_cumack(tp, rack, ae->ack, cts, to);
13588                                 if (rack->rc_dsack_round_seen) {
13589                                         /* Is the dsack round over? */
13590                                         if (SEQ_GEQ(ae->ack, rack->r_ctl.dsack_round_end)) {
13591                                                 /* Yes it is */
13592                                                 rack->rc_dsack_round_seen = 0;
13593                                                 rack_log_dsack_event(rack, 3, __LINE__, 0, 0);
13594                                         }
13595                                 }
13596                         }
13597                 }
13598                 /* And lets be sure to commit the rtt measurements for this ack */
13599                 tcp_rack_xmit_timer_commit(rack, tp);
13600 #ifdef TCP_ACCOUNTING
13601                 rdstc = get_cyclecount();
13602                 if (rdstc > ts_val) {
13603                         counter_u64_add(tcp_proc_time[ae->ack_val_set] , (rdstc - ts_val));
13604                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13605                                 tp->tcp_proc_time[ae->ack_val_set] += (rdstc - ts_val);
13606                                 if (ae->ack_val_set == ACK_CUMACK)
13607                                         tp->tcp_proc_time[CYC_HANDLE_MAP] += (rdstc - ts_val);
13608                         }
13609                 }
13610 #endif
13611         }
13612 #ifdef TCP_ACCOUNTING
13613         ts_val = get_cyclecount();
13614 #endif
13615         acked_amount = acked = (high_seq - tp->snd_una);
13616         if (acked) {
13617                 /*
13618                  * Clear the probe not answered flag
13619                  * since cum-ack moved forward.
13620                  */
13621                 rack->probe_not_answered = 0;
13622                 if (rack->sack_attack_disable == 0)
13623                         rack_do_decay(rack);
13624                 if (acked >= segsiz) {
13625                         /*
13626                          * You only get credit for
13627                          * MSS and greater (and you get extra
13628                          * credit for larger cum-ack moves).
13629                          */
13630                         int ac;
13631
13632                         ac = acked / segsiz;
13633                         rack->r_ctl.ack_count += ac;
13634                         counter_u64_add(rack_ack_total, ac);
13635                 }
13636                 if (rack->r_ctl.ack_count > 0xfff00000) {
13637                         /*
13638                          * reduce the number to keep us under
13639                          * a uint32_t.
13640                          */
13641                         rack->r_ctl.ack_count /= 2;
13642                         rack->r_ctl.sack_count /= 2;
13643                 }
13644                 if (tp->t_flags & TF_NEEDSYN) {
13645                         /*
13646                          * T/TCP: Connection was half-synchronized, and our SYN has
13647                          * been ACK'd (so connection is now fully synchronized).  Go
13648                          * to non-starred state, increment snd_una for ACK of SYN,
13649                          * and check if we can do window scaling.
13650                          */
13651                         tp->t_flags &= ~TF_NEEDSYN;
13652                         tp->snd_una++;
13653                         acked_amount = acked = (high_seq - tp->snd_una);
13654                 }
13655                 if (acked > sbavail(&so->so_snd))
13656                         acked_amount = sbavail(&so->so_snd);
13657 #ifdef NETFLIX_EXP_DETECTION
13658                 /*
13659                  * We only care on a cum-ack move if we are in a sack-disabled
13660                  * state. We have already added in to the ack_count, and we never
13661                  * would disable on a cum-ack move, so we only care to do the
13662                  * detection if it may "undo" it, i.e. we were in disabled already.
13663                  */
13664                 if (rack->sack_attack_disable)
13665                         rack_do_detection(tp, rack, acked_amount, segsiz);
13666 #endif
13667                 if (IN_FASTRECOVERY(tp->t_flags) &&
13668                     (rack->rack_no_prr == 0))
13669                         rack_update_prr(tp, rack, acked_amount, high_seq);
13670                 if (IN_RECOVERY(tp->t_flags)) {
13671                         if (SEQ_LT(high_seq, tp->snd_recover) &&
13672                             (SEQ_LT(high_seq, tp->snd_max))) {
13673                                 tcp_rack_partialack(tp);
13674                         } else {
13675                                 rack_post_recovery(tp, high_seq);
13676                                 recovery = 1;
13677                         }
13678                 }
13679                 /* Handle the rack-log-ack part (sendmap) */
13680                 if ((sbused(&so->so_snd) == 0) &&
13681                     (acked > acked_amount) &&
13682                     (tp->t_state >= TCPS_FIN_WAIT_1) &&
13683                     (tp->t_flags & TF_SENTFIN)) {
13684                         /*
13685                          * We must be sure our fin
13686                          * was sent and acked (we can be
13687                          * in FIN_WAIT_1 without having
13688                          * sent the fin).
13689                          */
13690                         ourfinisacked = 1;
13691                         /*
13692                          * Lets make sure snd_una is updated
13693                          * since most likely acked_amount = 0 (it
13694                          * should be).
13695                          */
13696                         tp->snd_una = high_seq;
13697                 }
13698                 /* Did we make a RTO error? */
13699                 if ((tp->t_flags & TF_PREVVALID) &&
13700                     ((tp->t_flags & TF_RCVD_TSTMP) == 0)) {
13701                         tp->t_flags &= ~TF_PREVVALID;
13702                         if (tp->t_rxtshift == 1 &&
13703                             (int)(ticks - tp->t_badrxtwin) < 0)
13704                                 rack_cong_signal(tp, CC_RTO_ERR, high_seq, __LINE__);
13705                 }
13706                 /* Handle the data in the socket buffer */
13707                 KMOD_TCPSTAT_ADD(tcps_rcvackpack, 1);
13708                 KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
13709                 if (acked_amount > 0) {
13710                         struct mbuf *mfree;
13711
13712                         rack_ack_received(tp, rack, high_seq, nsegs, CC_ACK, recovery);
13713                         SOCKBUF_LOCK(&so->so_snd);
13714                         mfree = sbcut_locked(&so->so_snd, acked_amount);
13715                         tp->snd_una = high_seq;
13716                         /* Note we want to hold the sb lock through the sendmap adjust */
13717                         rack_adjust_sendmap(rack, &so->so_snd, tp->snd_una);
13718                         /* Wake up the socket if we have room to write more */
13719                         rack_log_wakeup(tp,rack, &so->so_snd, acked, 2);
13720                         sowwakeup_locked(so);
13721                         m_freem(mfree);
13722                 }
13723                 /* update progress */
13724                 tp->t_acktime = ticks;
13725                 rack_log_progress_event(rack, tp, tp->t_acktime,
13726                                         PROGRESS_UPDATE, __LINE__);
13727                 /* Clear out shifts and such */
13728                 tp->t_rxtshift = 0;
13729                 RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
13730                                    rack_rto_min, rack_rto_max, rack->r_ctl.timer_slop);
13731                 rack->rc_tlp_in_progress = 0;
13732                 rack->r_ctl.rc_tlp_cnt_out = 0;
13733                 /* Send recover and snd_nxt must be dragged along */
13734                 if (SEQ_GT(tp->snd_una, tp->snd_recover))
13735                         tp->snd_recover = tp->snd_una;
13736                 if (SEQ_LT(tp->snd_nxt, tp->snd_una))
13737                         tp->snd_nxt = tp->snd_una;
13738                 /*
13739                  * If the RXT timer is running we want to
13740                  * stop it, so we can restart a TLP (or new RXT).
13741                  */
13742                 if (rack->r_ctl.rc_hpts_flags & PACE_TMR_RXT)
13743                         rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
13744 #ifdef NETFLIX_HTTP_LOGGING
13745                 tcp_http_check_for_comp(rack->rc_tp, high_seq);
13746 #endif
13747                 tp->snd_wl2 = high_seq;
13748                 tp->t_dupacks = 0;
13749                 if (under_pacing &&
13750                     (rack->use_fixed_rate == 0) &&
13751                     (rack->in_probe_rtt == 0) &&
13752                     rack->rc_gp_dyn_mul &&
13753                     rack->rc_always_pace) {
13754                         /* Check if we are dragging bottom */
13755                         rack_check_bottom_drag(tp, rack, so, acked);
13756                 }
13757                 if (tp->snd_una == tp->snd_max) {
13758                         tp->t_flags &= ~TF_PREVVALID;
13759                         rack->r_ctl.retran_during_recovery = 0;
13760                         rack->r_ctl.dsack_byte_cnt = 0;
13761                         rack->r_ctl.rc_went_idle_time = tcp_get_usecs(NULL);
13762                         if (rack->r_ctl.rc_went_idle_time == 0)
13763                                 rack->r_ctl.rc_went_idle_time = 1;
13764                         rack_log_progress_event(rack, tp, 0, PROGRESS_CLEAR, __LINE__);
13765                         if (sbavail(&tp->t_inpcb->inp_socket->so_snd) == 0)
13766                                 tp->t_acktime = 0;
13767                         /* Set so we might enter persists... */
13768                         rack->r_wanted_output = 1;
13769                         rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
13770                         sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
13771                         if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
13772                             (sbavail(&so->so_snd) == 0) &&
13773                             (tp->t_flags2 & TF2_DROP_AF_DATA)) {
13774                                 /*
13775                                  * The socket was gone and the
13776                                  * peer sent data (not now in the past), time to
13777                                  * reset him.
13778                                  */
13779                                 rack_timer_cancel(tp, rack, rack->r_ctl.rc_rcvtime, __LINE__);
13780                                 /* tcp_close will kill the inp pre-log the Reset */
13781                                 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
13782 #ifdef TCP_ACCOUNTING
13783                                 rdstc = get_cyclecount();
13784                                 if (rdstc > ts_val) {
13785                                         counter_u64_add(tcp_proc_time[ACK_CUMACK] , (rdstc - ts_val));
13786                                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13787                                                 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
13788                                                 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
13789                                         }
13790                                 }
13791 #endif
13792                                 m_freem(m);
13793                                 tp = tcp_close(tp);
13794                                 if (tp == NULL) {
13795 #ifdef TCP_ACCOUNTING
13796                                         sched_unpin();
13797 #endif
13798                                         return (1);
13799                                 }
13800                                 /*
13801                                  * We would normally do drop-with-reset which would
13802                                  * send back a reset. We can't since we don't have
13803                                  * all the needed bits. Instead lets arrange for
13804                                  * a call to tcp_output(). That way since we
13805                                  * are in the closed state we will generate a reset.
13806                                  *
13807                                  * Note if tcp_accounting is on we don't unpin since
13808                                  * we do that after the goto label.
13809                                  */
13810                                 goto send_out_a_rst;
13811                         }
13812                         if ((sbused(&so->so_snd) == 0) &&
13813                             (tp->t_state >= TCPS_FIN_WAIT_1) &&
13814                             (tp->t_flags & TF_SENTFIN)) {
13815                                 /*
13816                                  * If we can't receive any more data, then closing user can
13817                                  * proceed. Starting the timer is contrary to the
13818                                  * specification, but if we don't get a FIN we'll hang
13819                                  * forever.
13820                                  *
13821                                  */
13822                                 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
13823                                         soisdisconnected(so);
13824                                         tcp_timer_activate(tp, TT_2MSL,
13825                                                            (tcp_fast_finwait2_recycle ?
13826                                                             tcp_finwait2_timeout :
13827                                                             TP_MAXIDLE(tp)));
13828                                 }
13829                                 if (ourfinisacked == 0) {
13830                                         /*
13831                                          * We don't change to fin-wait-2 if we have our fin acked
13832                                          * which means we are probably in TCPS_CLOSING.
13833                                          */
13834                                         tcp_state_change(tp, TCPS_FIN_WAIT_2);
13835                                 }
13836                         }
13837                 }
13838                 /* Wake up the socket if we have room to write more */
13839                 if (sbavail(&so->so_snd)) {
13840                         rack->r_wanted_output = 1;
13841                         if (ctf_progress_timeout_check(tp, true)) {
13842                                 rack_log_progress_event((struct tcp_rack *)tp->t_fb_ptr,
13843                                                         tp, tick, PROGRESS_DROP, __LINE__);
13844                                 /*
13845                                  * We cheat here and don't send a RST, we should send one
13846                                  * when the pacer drops the connection.
13847                                  */
13848 #ifdef TCP_ACCOUNTING
13849                                 rdstc = get_cyclecount();
13850                                 if (rdstc > ts_val) {
13851                                         counter_u64_add(tcp_proc_time[ACK_CUMACK] , (rdstc - ts_val));
13852                                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13853                                                 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
13854                                                 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
13855                                         }
13856                                 }
13857                                 sched_unpin();
13858 #endif
13859                                 (void)tcp_drop(tp, ETIMEDOUT);
13860                                 m_freem(m);
13861                                 return (1);
13862                         }
13863                 }
13864                 if (ourfinisacked) {
13865                         switch(tp->t_state) {
13866                         case TCPS_CLOSING:
13867 #ifdef TCP_ACCOUNTING
13868                                 rdstc = get_cyclecount();
13869                                 if (rdstc > ts_val) {
13870                                         counter_u64_add(tcp_proc_time[ACK_CUMACK] ,
13871                                                         (rdstc - ts_val));
13872                                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13873                                                 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
13874                                                 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
13875                                         }
13876                                 }
13877                                 sched_unpin();
13878 #endif
13879                                 tcp_twstart(tp);
13880                                 m_freem(m);
13881                                 return (1);
13882                                 break;
13883                         case TCPS_LAST_ACK:
13884 #ifdef TCP_ACCOUNTING
13885                                 rdstc = get_cyclecount();
13886                                 if (rdstc > ts_val) {
13887                                         counter_u64_add(tcp_proc_time[ACK_CUMACK] ,
13888                                                         (rdstc - ts_val));
13889                                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13890                                                 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
13891                                                 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
13892                                         }
13893                                 }
13894                                 sched_unpin();
13895 #endif
13896                                 tp = tcp_close(tp);
13897                                 ctf_do_drop(m, tp);
13898                                 return (1);
13899                                 break;
13900                         case TCPS_FIN_WAIT_1:
13901 #ifdef TCP_ACCOUNTING
13902                                 rdstc = get_cyclecount();
13903                                 if (rdstc > ts_val) {
13904                                         counter_u64_add(tcp_proc_time[ACK_CUMACK] ,
13905                                                         (rdstc - ts_val));
13906                                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13907                                                 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
13908                                                 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
13909                                         }
13910                                 }
13911 #endif
13912                                 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
13913                                         soisdisconnected(so);
13914                                         tcp_timer_activate(tp, TT_2MSL,
13915                                                            (tcp_fast_finwait2_recycle ?
13916                                                             tcp_finwait2_timeout :
13917                                                             TP_MAXIDLE(tp)));
13918                                 }
13919                                 tcp_state_change(tp, TCPS_FIN_WAIT_2);
13920                                 break;
13921                         default:
13922                                 break;
13923                         }
13924                 }
13925                 if (rack->r_fast_output) {
13926                         /*
13927                          * We re doing fast output.. can we expand that?
13928                          */
13929                         rack_gain_for_fastoutput(rack, tp, so, acked_amount);
13930                 }
13931 #ifdef TCP_ACCOUNTING
13932                 rdstc = get_cyclecount();
13933                 if (rdstc > ts_val) {
13934                         counter_u64_add(tcp_proc_time[ACK_CUMACK] , (rdstc - ts_val));
13935                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13936                                 tp->tcp_proc_time[ACK_CUMACK] += (rdstc - ts_val);
13937                                 tp->tcp_proc_time[CYC_HANDLE_ACK] += (rdstc - ts_val);
13938                         }
13939                 }
13940
13941         } else if (win_up_req) {
13942                 rdstc = get_cyclecount();
13943                 if (rdstc > ts_val) {
13944                         counter_u64_add(tcp_proc_time[ACK_RWND] , (rdstc - ts_val));
13945                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
13946                                 tp->tcp_proc_time[ACK_RWND] += (rdstc - ts_val);
13947                         }
13948                 }
13949 #endif
13950         }
13951         /* Now is there a next packet, if so we are done */
13952         m_freem(m);
13953         did_out = 0;
13954         if (nxt_pkt) {
13955 #ifdef TCP_ACCOUNTING
13956                 sched_unpin();
13957 #endif
13958                 rack_log_doseg_done(rack, cts, nxt_pkt, did_out, 5, nsegs);
13959                 return (0);
13960         }
13961         rack_handle_might_revert(tp, rack);
13962         ctf_calc_rwin(so, tp);
13963         if ((rack->r_wanted_output != 0) || (rack->r_fast_output != 0)) {
13964         send_out_a_rst:
13965                 if (tcp_output(tp) < 0) {
13966 #ifdef TCP_ACCOUNTING
13967                         sched_unpin();
13968 #endif
13969                         return (1);
13970                 }
13971                 did_out = 1;
13972         }
13973         rack_free_trim(rack);
13974 #ifdef TCP_ACCOUNTING
13975         sched_unpin();
13976 #endif
13977         rack_timer_audit(tp, rack, &so->so_snd);
13978         rack_log_doseg_done(rack, cts, nxt_pkt, did_out, 6, nsegs);
13979         return (0);
13980 }
13981
13982
13983 static int
13984 rack_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
13985     struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos,
13986     int32_t nxt_pkt, struct timeval *tv)
13987 {
13988 #ifdef TCP_ACCOUNTING
13989         uint64_t ts_val;
13990 #endif
13991         int32_t thflags, retval, did_out = 0;
13992         int32_t way_out = 0;
13993         /*
13994          * cts - is the current time from tv (caller gets ts) in microseconds.
13995          * ms_cts - is the current time from tv in milliseconds.
13996          * us_cts - is the time that LRO or hardware actually got the packet in microseconds.
13997          */
13998         uint32_t cts, us_cts, ms_cts;
13999         uint32_t tiwin, high_seq;
14000         struct timespec ts;
14001         struct tcpopt to;
14002         struct tcp_rack *rack;
14003         struct rack_sendmap *rsm;
14004         int32_t prev_state = 0;
14005 #ifdef TCP_ACCOUNTING
14006         int ack_val_set = 0xf;
14007 #endif
14008         int nsegs;
14009         /*
14010          * tv passed from common code is from either M_TSTMP_LRO or
14011          * tcp_get_usecs() if no LRO m_pkthdr timestamp is present.
14012          */
14013         rack = (struct tcp_rack *)tp->t_fb_ptr;
14014         if (m->m_flags & M_ACKCMP) {
14015                 return (rack_do_compressed_ack_processing(tp, so, m, nxt_pkt, tv));
14016         }
14017         if (m->m_flags & M_ACKCMP) {
14018                 panic("Impossible reach m has ackcmp? m:%p tp:%p", m, tp);
14019         }
14020         cts = tcp_tv_to_usectick(tv);
14021         ms_cts =  tcp_tv_to_mssectick(tv);
14022         nsegs = m->m_pkthdr.lro_nsegs;
14023         counter_u64_add(rack_proc_non_comp_ack, 1);
14024         thflags = tcp_get_flags(th);
14025 #ifdef TCP_ACCOUNTING
14026         sched_pin();
14027         if (thflags & TH_ACK)
14028                 ts_val = get_cyclecount();
14029 #endif
14030         if ((m->m_flags & M_TSTMP) ||
14031             (m->m_flags & M_TSTMP_LRO)) {
14032                 mbuf_tstmp2timespec(m, &ts);
14033                 rack->r_ctl.act_rcv_time.tv_sec = ts.tv_sec;
14034                 rack->r_ctl.act_rcv_time.tv_usec = ts.tv_nsec/1000;
14035         } else
14036                 rack->r_ctl.act_rcv_time = *tv;
14037         kern_prefetch(rack, &prev_state);
14038         prev_state = 0;
14039         /*
14040          * Unscale the window into a 32-bit value. For the SYN_SENT state
14041          * the scale is zero.
14042          */
14043         tiwin = th->th_win << tp->snd_scale;
14044 #ifdef TCP_ACCOUNTING
14045         if (thflags & TH_ACK) {
14046                 /*
14047                  * We have a tradeoff here. We can either do what we are
14048                  * doing i.e. pinning to this CPU and then doing the accounting
14049                  * <or> we could do a critical enter, setup the rdtsc and cpu
14050                  * as in below, and then validate we are on the same CPU on
14051                  * exit. I have choosen to not do the critical enter since
14052                  * that often will gain you a context switch, and instead lock
14053                  * us (line above this if) to the same CPU with sched_pin(). This
14054                  * means we may be context switched out for a higher priority
14055                  * interupt but we won't be moved to another CPU.
14056                  *
14057                  * If this occurs (which it won't very often since we most likely
14058                  * are running this code in interupt context and only a higher
14059                  * priority will bump us ... clock?) we will falsely add in
14060                  * to the time the interupt processing time plus the ack processing
14061                  * time. This is ok since its a rare event.
14062                  */
14063                 ack_val_set = tcp_do_ack_accounting(tp, th, &to, tiwin,
14064                                                     ctf_fixed_maxseg(tp));
14065         }
14066 #endif
14067         /*
14068          * Parse options on any incoming segment.
14069          */
14070         memset(&to, 0, sizeof(to));
14071         tcp_dooptions(&to, (u_char *)(th + 1),
14072             (th->th_off << 2) - sizeof(struct tcphdr),
14073             (thflags & TH_SYN) ? TO_SYN : 0);
14074         NET_EPOCH_ASSERT();
14075         INP_WLOCK_ASSERT(tp->t_inpcb);
14076         KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",
14077             __func__));
14078         KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",
14079             __func__));
14080         if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
14081             (tp->t_flags & TF_GPUTINPROG)) {
14082                 /*
14083                  * We have a goodput in progress
14084                  * and we have entered a late state.
14085                  * Do we have enough data in the sb
14086                  * to handle the GPUT request?
14087                  */
14088                 uint32_t bytes;
14089
14090                 bytes = tp->gput_ack - tp->gput_seq;
14091                 if (SEQ_GT(tp->gput_seq, tp->snd_una))
14092                         bytes += tp->gput_seq - tp->snd_una;
14093                 if (bytes > sbavail(&tp->t_inpcb->inp_socket->so_snd)) {
14094                         /*
14095                          * There are not enough bytes in the socket
14096                          * buffer that have been sent to cover this
14097                          * measurement. Cancel it.
14098                          */
14099                         rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
14100                                                    rack->r_ctl.rc_gp_srtt /*flex1*/,
14101                                                    tp->gput_seq,
14102                                                    0, 0, 18, __LINE__, NULL, 0);
14103                         tp->t_flags &= ~TF_GPUTINPROG;
14104                 }
14105         }
14106         high_seq = th->th_ack;
14107         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
14108                 union tcp_log_stackspecific log;
14109                 struct timeval ltv;
14110 #ifdef NETFLIX_HTTP_LOGGING
14111                 struct http_sendfile_track *http_req;
14112
14113                 if (SEQ_GT(th->th_ack, tp->snd_una)) {
14114                         http_req = tcp_http_find_req_for_seq(tp, (th->th_ack-1));
14115                 } else {
14116                         http_req = tcp_http_find_req_for_seq(tp, th->th_ack);
14117                 }
14118 #endif
14119                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
14120                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
14121                 if (rack->rack_no_prr == 0)
14122                         log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
14123                 else
14124                         log.u_bbr.flex1 = 0;
14125                 log.u_bbr.use_lt_bw = rack->r_ent_rec_ns;
14126                 log.u_bbr.use_lt_bw <<= 1;
14127                 log.u_bbr.use_lt_bw |= rack->r_might_revert;
14128                 log.u_bbr.flex2 = rack->r_ctl.rc_num_maps_alloced;
14129                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
14130                 log.u_bbr.pkts_out = rack->rc_tp->t_maxseg;
14131                 log.u_bbr.flex3 = m->m_flags;
14132                 log.u_bbr.flex4 = rack->r_ctl.rc_hpts_flags;
14133                 log.u_bbr.lost = thflags;
14134                 log.u_bbr.pacing_gain = 0x1;
14135 #ifdef TCP_ACCOUNTING
14136                 log.u_bbr.cwnd_gain = ack_val_set;
14137 #endif
14138                 log.u_bbr.flex7 = 2;
14139                 if (m->m_flags & M_TSTMP) {
14140                         /* Record the hardware timestamp if present */
14141                         mbuf_tstmp2timespec(m, &ts);
14142                         ltv.tv_sec = ts.tv_sec;
14143                         ltv.tv_usec = ts.tv_nsec / 1000;
14144                         log.u_bbr.lt_epoch = tcp_tv_to_usectick(&ltv);
14145                 } else if (m->m_flags & M_TSTMP_LRO) {
14146                         /* Record the LRO the arrival timestamp */
14147                         mbuf_tstmp2timespec(m, &ts);
14148                         ltv.tv_sec = ts.tv_sec;
14149                         ltv.tv_usec = ts.tv_nsec / 1000;
14150                         log.u_bbr.flex5 = tcp_tv_to_usectick(&ltv);
14151                 }
14152                 log.u_bbr.timeStamp = tcp_get_usecs(&ltv);
14153                 /* Log the rcv time */
14154                 log.u_bbr.delRate = m->m_pkthdr.rcv_tstmp;
14155 #ifdef NETFLIX_HTTP_LOGGING
14156                 log.u_bbr.applimited = tp->t_http_closed;
14157                 log.u_bbr.applimited <<= 8;
14158                 log.u_bbr.applimited |= tp->t_http_open;
14159                 log.u_bbr.applimited <<= 8;
14160                 log.u_bbr.applimited |= tp->t_http_req;
14161                 if (http_req) {
14162                         /* Copy out any client req info */
14163                         /* seconds */
14164                         log.u_bbr.pkt_epoch = (http_req->localtime / HPTS_USEC_IN_SEC);
14165                         /* useconds */
14166                         log.u_bbr.delivered = (http_req->localtime % HPTS_USEC_IN_SEC);
14167                         log.u_bbr.rttProp = http_req->timestamp;
14168                         log.u_bbr.cur_del_rate = http_req->start;
14169                         if (http_req->flags & TCP_HTTP_TRACK_FLG_OPEN) {
14170                                 log.u_bbr.flex8 |= 1;
14171                         } else {
14172                                 log.u_bbr.flex8 |= 2;
14173                                 log.u_bbr.bw_inuse = http_req->end;
14174                         }
14175                         log.u_bbr.flex6 = http_req->start_seq;
14176                         if (http_req->flags & TCP_HTTP_TRACK_FLG_COMP) {
14177                                 log.u_bbr.flex8 |= 4;
14178                                 log.u_bbr.epoch = http_req->end_seq;
14179                         }
14180                 }
14181 #endif
14182                 TCP_LOG_EVENTP(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_IN, 0,
14183                     tlen, &log, true, &ltv);
14184         }
14185         if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) {
14186                 way_out = 4;
14187                 retval = 0;
14188                 m_freem(m);
14189                 goto done_with_input;
14190         }
14191         /*
14192          * If a segment with the ACK-bit set arrives in the SYN-SENT state
14193          * check SEQ.ACK first as described on page 66 of RFC 793, section 3.9.
14194          */
14195         if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) &&
14196             (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) {
14197                 tcp_log_end_status(tp, TCP_EI_STATUS_RST_IN_FRONT);
14198                 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
14199 #ifdef TCP_ACCOUNTING
14200                 sched_unpin();
14201 #endif
14202                 return (1);
14203         }
14204         /*
14205          * If timestamps were negotiated during SYN/ACK and a
14206          * segment without a timestamp is received, silently drop
14207          * the segment, unless it is a RST segment or missing timestamps are
14208          * tolerated.
14209          * See section 3.2 of RFC 7323.
14210          */
14211         if ((tp->t_flags & TF_RCVD_TSTMP) && !(to.to_flags & TOF_TS) &&
14212             ((thflags & TH_RST) == 0) && (V_tcp_tolerate_missing_ts == 0)) {
14213                 way_out = 5;
14214                 retval = 0;
14215                 m_freem(m);
14216                 goto done_with_input;
14217         }
14218
14219         /*
14220          * Segment received on connection. Reset idle time and keep-alive
14221          * timer. XXX: This should be done after segment validation to
14222          * ignore broken/spoofed segs.
14223          */
14224         if  (tp->t_idle_reduce &&
14225              (tp->snd_max == tp->snd_una) &&
14226              (TICKS_2_USEC(ticks - tp->t_rcvtime) >= tp->t_rxtcur)) {
14227                 counter_u64_add(rack_input_idle_reduces, 1);
14228                 rack_cc_after_idle(rack, tp);
14229         }
14230         tp->t_rcvtime = ticks;
14231 #ifdef STATS
14232         stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);
14233 #endif
14234         if (tiwin > rack->r_ctl.rc_high_rwnd)
14235                 rack->r_ctl.rc_high_rwnd = tiwin;
14236         /*
14237          * TCP ECN processing. XXXJTL: If we ever use ECN, we need to move
14238          * this to occur after we've validated the segment.
14239          */
14240         if (tcp_ecn_input_segment(tp, thflags, iptos))
14241                 rack_cong_signal(tp, CC_ECN, th->th_ack, __LINE__);
14242
14243         /*
14244          * If echoed timestamp is later than the current time, fall back to
14245          * non RFC1323 RTT calculation.  Normalize timestamp if syncookies
14246          * were used when this connection was established.
14247          */
14248         if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) {
14249                 to.to_tsecr -= tp->ts_offset;
14250                 if (TSTMP_GT(to.to_tsecr, ms_cts))
14251                         to.to_tsecr = 0;
14252         }
14253
14254         /*
14255          * If its the first time in we need to take care of options and
14256          * verify we can do SACK for rack!
14257          */
14258         if (rack->r_state == 0) {
14259                 /* Should be init'd by rack_init() */
14260                 KASSERT(rack->rc_inp != NULL,
14261                     ("%s: rack->rc_inp unexpectedly NULL", __func__));
14262                 if (rack->rc_inp == NULL) {
14263                         rack->rc_inp = tp->t_inpcb;
14264                 }
14265
14266                 /*
14267                  * Process options only when we get SYN/ACK back. The SYN
14268                  * case for incoming connections is handled in tcp_syncache.
14269                  * According to RFC1323 the window field in a SYN (i.e., a
14270                  * <SYN> or <SYN,ACK>) segment itself is never scaled. XXX
14271                  * this is traditional behavior, may need to be cleaned up.
14272                  */
14273                 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
14274                         /* Handle parallel SYN for ECN */
14275                         tcp_ecn_input_parallel_syn(tp, thflags, iptos);
14276                         if ((to.to_flags & TOF_SCALE) &&
14277                             (tp->t_flags & TF_REQ_SCALE)) {
14278                                 tp->t_flags |= TF_RCVD_SCALE;
14279                                 tp->snd_scale = to.to_wscale;
14280                         } else
14281                                 tp->t_flags &= ~TF_REQ_SCALE;
14282                         /*
14283                          * Initial send window.  It will be updated with the
14284                          * next incoming segment to the scaled value.
14285                          */
14286                         tp->snd_wnd = th->th_win;
14287                         rack_validate_fo_sendwin_up(tp, rack);
14288                         if ((to.to_flags & TOF_TS) &&
14289                             (tp->t_flags & TF_REQ_TSTMP)) {
14290                                 tp->t_flags |= TF_RCVD_TSTMP;
14291                                 tp->ts_recent = to.to_tsval;
14292                                 tp->ts_recent_age = cts;
14293                         } else
14294                                 tp->t_flags &= ~TF_REQ_TSTMP;
14295                         if (to.to_flags & TOF_MSS) {
14296                                 tcp_mss(tp, to.to_mss);
14297                         }
14298                         if ((tp->t_flags & TF_SACK_PERMIT) &&
14299                             (to.to_flags & TOF_SACKPERM) == 0)
14300                                 tp->t_flags &= ~TF_SACK_PERMIT;
14301                         if (IS_FASTOPEN(tp->t_flags)) {
14302                                 if (to.to_flags & TOF_FASTOPEN) {
14303                                         uint16_t mss;
14304
14305                                         if (to.to_flags & TOF_MSS)
14306                                                 mss = to.to_mss;
14307                                         else
14308                                                 if ((tp->t_inpcb->inp_vflag & INP_IPV6) != 0)
14309                                                         mss = TCP6_MSS;
14310                                                 else
14311                                                         mss = TCP_MSS;
14312                                         tcp_fastopen_update_cache(tp, mss,
14313                                             to.to_tfo_len, to.to_tfo_cookie);
14314                                 } else
14315                                         tcp_fastopen_disable_path(tp);
14316                         }
14317                 }
14318                 /*
14319                  * At this point we are at the initial call. Here we decide
14320                  * if we are doing RACK or not. We do this by seeing if
14321                  * TF_SACK_PERMIT is set and the sack-not-required is clear.
14322                  * The code now does do dup-ack counting so if you don't
14323                  * switch back you won't get rack & TLP, but you will still
14324                  * get this stack.
14325                  */
14326
14327                 if ((rack_sack_not_required == 0) &&
14328                     ((tp->t_flags & TF_SACK_PERMIT) == 0)) {
14329                         tcp_switch_back_to_default(tp);
14330                         (*tp->t_fb->tfb_tcp_do_segment) (m, th, so, tp, drop_hdrlen,
14331                             tlen, iptos);
14332 #ifdef TCP_ACCOUNTING
14333                         sched_unpin();
14334 #endif
14335                         return (1);
14336                 }
14337                 tcp_set_hpts(tp->t_inpcb);
14338                 sack_filter_clear(&rack->r_ctl.rack_sf, th->th_ack);
14339         }
14340         if (thflags & TH_FIN)
14341                 tcp_log_end_status(tp, TCP_EI_STATUS_CLIENT_FIN);
14342         us_cts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time);
14343         if ((rack->rc_gp_dyn_mul) &&
14344             (rack->use_fixed_rate == 0) &&
14345             (rack->rc_always_pace)) {
14346                 /* Check in on probertt */
14347                 rack_check_probe_rtt(rack, us_cts);
14348         }
14349         rack_clear_rate_sample(rack);
14350         if ((rack->forced_ack) &&
14351             ((tcp_get_flags(th) & TH_RST) == 0)) {
14352                 rack_handle_probe_response(rack, tiwin, us_cts);
14353         }
14354         /*
14355          * This is the one exception case where we set the rack state
14356          * always. All other times (timers etc) we must have a rack-state
14357          * set (so we assure we have done the checks above for SACK).
14358          */
14359         rack->r_ctl.rc_rcvtime = cts;
14360         if (rack->r_state != tp->t_state)
14361                 rack_set_state(tp, rack);
14362         if (SEQ_GT(th->th_ack, tp->snd_una) &&
14363             (rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree)) != NULL)
14364                 kern_prefetch(rsm, &prev_state);
14365         prev_state = rack->r_state;
14366         retval = (*rack->r_substate) (m, th, so,
14367             tp, &to, drop_hdrlen,
14368             tlen, tiwin, thflags, nxt_pkt, iptos);
14369 #ifdef INVARIANTS
14370         if ((retval == 0) &&
14371             (tp->t_inpcb == NULL)) {
14372                 panic("retval:%d tp:%p t_inpcb:NULL state:%d",
14373                     retval, tp, prev_state);
14374         }
14375 #endif
14376         if (retval == 0) {
14377                 /*
14378                  * If retval is 1 the tcb is unlocked and most likely the tp
14379                  * is gone.
14380                  */
14381                 INP_WLOCK_ASSERT(tp->t_inpcb);
14382                 if ((rack->rc_gp_dyn_mul) &&
14383                     (rack->rc_always_pace) &&
14384                     (rack->use_fixed_rate == 0) &&
14385                     rack->in_probe_rtt &&
14386                     (rack->r_ctl.rc_time_probertt_starts == 0)) {
14387                         /*
14388                          * If we are going for target, lets recheck before
14389                          * we output.
14390                          */
14391                         rack_check_probe_rtt(rack, us_cts);
14392                 }
14393                 if (rack->set_pacing_done_a_iw == 0) {
14394                         /* How much has been acked? */
14395                         if ((tp->snd_una - tp->iss) > (ctf_fixed_maxseg(tp) * 10)) {
14396                                 /* We have enough to set in the pacing segment size */
14397                                 rack->set_pacing_done_a_iw = 1;
14398                                 rack_set_pace_segments(tp, rack, __LINE__, NULL);
14399                         }
14400                 }
14401                 tcp_rack_xmit_timer_commit(rack, tp);
14402 #ifdef TCP_ACCOUNTING
14403                 /*
14404                  * If we set the ack_val_se to what ack processing we are doing
14405                  * we also want to track how many cycles we burned. Note
14406                  * the bits after tcp_output we let be "free". This is because
14407                  * we are also tracking the tcp_output times as well. Note the
14408                  * use of 0xf here since we only have 11 counter (0 - 0xa) and
14409                  * 0xf cannot be returned and is what we initialize it too to
14410                  * indicate we are not doing the tabulations.
14411                  */
14412                 if (ack_val_set != 0xf) {
14413                         uint64_t crtsc;
14414
14415                         crtsc = get_cyclecount();
14416                         counter_u64_add(tcp_proc_time[ack_val_set] , (crtsc - ts_val));
14417                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
14418                                 tp->tcp_proc_time[ack_val_set] += (crtsc - ts_val);
14419                         }
14420                 }
14421 #endif
14422                 if (nxt_pkt == 0) {
14423                         if ((rack->r_wanted_output != 0) || (rack->r_fast_output != 0)) {
14424 do_output_now:
14425                                 if (tcp_output(tp) < 0)
14426                                         return (1);
14427                                 did_out = 1;
14428                         }
14429                         rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
14430                         rack_free_trim(rack);
14431                 }
14432                 /* Update any rounds needed */
14433                 if (rack_verbose_logging &&  (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
14434                         union tcp_log_stackspecific log;
14435                         struct timeval tv;
14436
14437                         memset(&log.u_bbr, 0, sizeof(log.u_bbr));
14438                         log.u_bbr.timeStamp = tcp_get_usecs(&tv);
14439                         log.u_bbr.flex1 = high_seq;
14440                         log.u_bbr.flex2 = rack->r_ctl.roundends;
14441                         log.u_bbr.flex3 = rack->r_ctl.current_round;
14442                         log.u_bbr.rttProp = (uint64_t)CC_ALGO(tp)->newround;
14443                         log.u_bbr.flex8 = 9;
14444                         tcp_log_event_(tp, NULL, NULL, NULL, BBR_LOG_CWND, 0,
14445                                        0, &log, false, NULL, NULL, 0, &tv);
14446                 }
14447                 /*
14448                  * The draft (v3) calls for us to use SEQ_GEQ, but that
14449                  * causes issues when we are just going app limited. Lets
14450                  * instead use SEQ_GT <or> where its equal but more data
14451                  * is outstanding.
14452                  */
14453                 if ((SEQ_GT(tp->snd_una, rack->r_ctl.roundends)) ||
14454                     ((tp->snd_una == rack->r_ctl.roundends) && SEQ_GT(tp->snd_max, tp->snd_una))) {
14455                         rack->r_ctl.current_round++;
14456                         rack->r_ctl.roundends = tp->snd_max;
14457                         if (CC_ALGO(tp)->newround != NULL) {
14458                                 CC_ALGO(tp)->newround(tp->ccv, rack->r_ctl.current_round);
14459                         }
14460                 }
14461                 if ((nxt_pkt == 0) &&
14462                     ((rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&
14463                     (SEQ_GT(tp->snd_max, tp->snd_una) ||
14464                      (tp->t_flags & TF_DELACK) ||
14465                      ((V_tcp_always_keepalive || rack->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&
14466                       (tp->t_state <= TCPS_CLOSING)))) {
14467                         /* We could not send (probably in the hpts but stopped the timer earlier)? */
14468                         if ((tp->snd_max == tp->snd_una) &&
14469                             ((tp->t_flags & TF_DELACK) == 0) &&
14470                             (tcp_in_hpts(rack->rc_inp)) &&
14471                             (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
14472                                 /* keep alive not needed if we are hptsi output yet */
14473                                 ;
14474                         } else {
14475                                 int late = 0;
14476                                 if (tcp_in_hpts(rack->rc_inp)) {
14477                                         if (rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
14478                                                 us_cts = tcp_get_usecs(NULL);
14479                                                 if (TSTMP_GT(rack->r_ctl.rc_last_output_to, us_cts)) {
14480                                                         rack->r_early = 1;
14481                                                         rack->r_ctl.rc_agg_early += (rack->r_ctl.rc_last_output_to - us_cts);
14482                                                 } else
14483                                                         late = 1;
14484                                                 rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
14485                                         }
14486                                         tcp_hpts_remove(tp->t_inpcb);
14487                                 }
14488                                 if (late && (did_out == 0)) {
14489                                         /*
14490                                          * We are late in the sending
14491                                          * and we did not call the output
14492                                          * (this probably should not happen).
14493                                          */
14494                                         goto do_output_now;
14495                                 }
14496                                 rack_start_hpts_timer(rack, tp, tcp_get_usecs(NULL), 0, 0, 0);
14497                         }
14498                         way_out = 1;
14499                 } else if (nxt_pkt == 0) {
14500                         /* Do we have the correct timer running? */
14501                         rack_timer_audit(tp, rack, &so->so_snd);
14502                         way_out = 2;
14503                 }
14504         done_with_input:
14505                 rack_log_doseg_done(rack, cts, nxt_pkt, did_out, way_out, max(1, nsegs));
14506                 if (did_out)
14507                         rack->r_wanted_output = 0;
14508 #ifdef INVARIANTS
14509                 if (tp->t_inpcb == NULL) {
14510                         panic("OP:%d retval:%d tp:%p t_inpcb:NULL state:%d",
14511                               did_out,
14512                               retval, tp, prev_state);
14513                 }
14514 #endif
14515 #ifdef TCP_ACCOUNTING
14516         } else {
14517                 /*
14518                  * Track the time (see above).
14519                  */
14520                 if (ack_val_set != 0xf) {
14521                         uint64_t crtsc;
14522
14523                         crtsc = get_cyclecount();
14524                         counter_u64_add(tcp_proc_time[ack_val_set] , (crtsc - ts_val));
14525                         /*
14526                          * Note we *DO NOT* increment the per-tcb counters since
14527                          * in the else the TP may be gone!!
14528                          */
14529                 }
14530 #endif
14531         }
14532 #ifdef TCP_ACCOUNTING
14533         sched_unpin();
14534 #endif
14535         return (retval);
14536 }
14537
14538 void
14539 rack_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
14540     struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos)
14541 {
14542         struct timeval tv;
14543
14544         /* First lets see if we have old packets */
14545         if (tp->t_in_pkt) {
14546                 if (ctf_do_queued_segments(so, tp, 1)) {
14547                         m_freem(m);
14548                         return;
14549                 }
14550         }
14551         if (m->m_flags & M_TSTMP_LRO) {
14552                 tv.tv_sec = m->m_pkthdr.rcv_tstmp /1000000000;
14553                 tv.tv_usec = (m->m_pkthdr.rcv_tstmp % 1000000000)/1000;
14554         } else {
14555                 /* Should not be should we kassert instead? */
14556                 tcp_get_usecs(&tv);
14557         }
14558         if (rack_do_segment_nounlock(m, th, so, tp,
14559                                      drop_hdrlen, tlen, iptos, 0, &tv) == 0) {
14560                 INP_WUNLOCK(tp->t_inpcb);
14561         }
14562 }
14563
14564 struct rack_sendmap *
14565 tcp_rack_output(struct tcpcb *tp, struct tcp_rack *rack, uint32_t tsused)
14566 {
14567         struct rack_sendmap *rsm = NULL;
14568         int32_t idx;
14569         uint32_t srtt = 0, thresh = 0, ts_low = 0;
14570
14571         /* Return the next guy to be re-transmitted */
14572         if (RB_EMPTY(&rack->r_ctl.rc_mtree)) {
14573                 return (NULL);
14574         }
14575         if (tp->t_flags & TF_SENTFIN) {
14576                 /* retran the end FIN? */
14577                 return (NULL);
14578         }
14579         /* ok lets look at this one */
14580         rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
14581         if (rack->r_must_retran && rsm && (rsm->r_flags & RACK_MUST_RXT)) {
14582                 return (rsm);
14583         }
14584         if (rsm && ((rsm->r_flags & RACK_ACKED) == 0)) {
14585                 goto check_it;
14586         }
14587         rsm = rack_find_lowest_rsm(rack);
14588         if (rsm == NULL) {
14589                 return (NULL);
14590         }
14591 check_it:
14592         if (((rack->rc_tp->t_flags & TF_SACK_PERMIT) == 0) &&
14593             (rsm->r_dupack >= DUP_ACK_THRESHOLD)) {
14594                 /*
14595                  * No sack so we automatically do the 3 strikes and
14596                  * retransmit (no rack timer would be started).
14597                  */
14598
14599                 return (rsm);
14600         }
14601         if (rsm->r_flags & RACK_ACKED) {
14602                 return (NULL);
14603         }
14604         if (((rsm->r_flags & RACK_SACK_PASSED) == 0) &&
14605             (rsm->r_dupack < DUP_ACK_THRESHOLD)) {
14606                 /* Its not yet ready */
14607                 return (NULL);
14608         }
14609         srtt = rack_grab_rtt(tp, rack);
14610         idx = rsm->r_rtr_cnt - 1;
14611         ts_low = (uint32_t)rsm->r_tim_lastsent[idx];
14612         thresh = rack_calc_thresh_rack(rack, srtt, tsused);
14613         if ((tsused == ts_low) ||
14614             (TSTMP_LT(tsused, ts_low))) {
14615                 /* No time since sending */
14616                 return (NULL);
14617         }
14618         if ((tsused - ts_low) < thresh) {
14619                 /* It has not been long enough yet */
14620                 return (NULL);
14621         }
14622         if ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||
14623             ((rsm->r_flags & RACK_SACK_PASSED) &&
14624              (rack->sack_attack_disable == 0))) {
14625                 /*
14626                  * We have passed the dup-ack threshold <or>
14627                  * a SACK has indicated this is missing.
14628                  * Note that if you are a declared attacker
14629                  * it is only the dup-ack threshold that
14630                  * will cause retransmits.
14631                  */
14632                 /* log retransmit reason */
14633                 rack_log_retran_reason(rack, rsm, (tsused - ts_low), thresh, 1);
14634                 rack->r_fast_output = 0;
14635                 return (rsm);
14636         }
14637         return (NULL);
14638 }
14639
14640 static void
14641 rack_log_pacing_delay_calc(struct tcp_rack *rack, uint32_t len, uint32_t slot,
14642                            uint64_t bw_est, uint64_t bw, uint64_t len_time, int method,
14643                            int line, struct rack_sendmap *rsm, uint8_t quality)
14644 {
14645         if (rack->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
14646                 union tcp_log_stackspecific log;
14647                 struct timeval tv;
14648
14649                 memset(&log, 0, sizeof(log));
14650                 log.u_bbr.flex1 = slot;
14651                 log.u_bbr.flex2 = len;
14652                 log.u_bbr.flex3 = rack->r_ctl.rc_pace_min_segs;
14653                 log.u_bbr.flex4 = rack->r_ctl.rc_pace_max_segs;
14654                 log.u_bbr.flex5 = rack->r_ctl.rack_per_of_gp_ss;
14655                 log.u_bbr.flex6 = rack->r_ctl.rack_per_of_gp_ca;
14656                 log.u_bbr.use_lt_bw = rack->rc_ack_can_sendout_data;
14657                 log.u_bbr.use_lt_bw <<= 1;
14658                 log.u_bbr.use_lt_bw |= rack->r_late;
14659                 log.u_bbr.use_lt_bw <<= 1;
14660                 log.u_bbr.use_lt_bw |= rack->r_early;
14661                 log.u_bbr.use_lt_bw <<= 1;
14662                 log.u_bbr.use_lt_bw |= rack->app_limited_needs_set;
14663                 log.u_bbr.use_lt_bw <<= 1;
14664                 log.u_bbr.use_lt_bw |= rack->rc_gp_filled;
14665                 log.u_bbr.use_lt_bw <<= 1;
14666                 log.u_bbr.use_lt_bw |= rack->measure_saw_probe_rtt;
14667                 log.u_bbr.use_lt_bw <<= 1;
14668                 log.u_bbr.use_lt_bw |= rack->in_probe_rtt;
14669                 log.u_bbr.use_lt_bw <<= 1;
14670                 log.u_bbr.use_lt_bw |= rack->gp_ready;
14671                 log.u_bbr.pkt_epoch = line;
14672                 log.u_bbr.epoch = rack->r_ctl.rc_agg_delayed;
14673                 log.u_bbr.lt_epoch = rack->r_ctl.rc_agg_early;
14674                 log.u_bbr.applimited = rack->r_ctl.rack_per_of_gp_rec;
14675                 log.u_bbr.bw_inuse = bw_est;
14676                 log.u_bbr.delRate = bw;
14677                 if (rack->r_ctl.gp_bw == 0)
14678                         log.u_bbr.cur_del_rate = 0;
14679                 else
14680                         log.u_bbr.cur_del_rate = rack_get_bw(rack);
14681                 log.u_bbr.rttProp = len_time;
14682                 log.u_bbr.pkts_out = rack->r_ctl.rc_rack_min_rtt;
14683                 log.u_bbr.lost = rack->r_ctl.rc_probertt_sndmax_atexit;
14684                 log.u_bbr.pacing_gain = rack_get_output_gain(rack, rsm);
14685                 if (rack->r_ctl.cwnd_to_use < rack->rc_tp->snd_ssthresh) {
14686                         /* We are in slow start */
14687                         log.u_bbr.flex7 = 1;
14688                 } else {
14689                         /* we are on congestion avoidance */
14690                         log.u_bbr.flex7 = 0;
14691                 }
14692                 log.u_bbr.flex8 = method;
14693                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
14694                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
14695                 log.u_bbr.cwnd_gain = rack->rc_gp_saw_rec;
14696                 log.u_bbr.cwnd_gain <<= 1;
14697                 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ss;
14698                 log.u_bbr.cwnd_gain <<= 1;
14699                 log.u_bbr.cwnd_gain |= rack->rc_gp_saw_ca;
14700                 log.u_bbr.bbr_substate = quality;
14701                 TCP_LOG_EVENTP(rack->rc_tp, NULL,
14702                     &rack->rc_inp->inp_socket->so_rcv,
14703                     &rack->rc_inp->inp_socket->so_snd,
14704                     BBR_LOG_HPTSI_CALC, 0,
14705                     0, &log, false, &tv);
14706         }
14707 }
14708
14709 static uint32_t
14710 rack_get_pacing_len(struct tcp_rack *rack, uint64_t bw, uint32_t mss)
14711 {
14712         uint32_t new_tso, user_max;
14713
14714         user_max = rack->rc_user_set_max_segs * mss;
14715         if (rack->rc_force_max_seg) {
14716                 return (user_max);
14717         }
14718         if (rack->use_fixed_rate &&
14719             ((rack->r_ctl.crte == NULL) ||
14720              (bw != rack->r_ctl.crte->rate))) {
14721                 /* Use the user mss since we are not exactly matched */
14722                 return (user_max);
14723         }
14724         new_tso = tcp_get_pacing_burst_size(rack->rc_tp, bw, mss, rack_pace_one_seg, rack->r_ctl.crte, NULL);
14725         if (new_tso > user_max)
14726                 new_tso = user_max;
14727         return (new_tso);
14728 }
14729
14730 static int32_t
14731 pace_to_fill_cwnd(struct tcp_rack *rack, int32_t slot, uint32_t len, uint32_t segsiz, int *capped, uint64_t *rate_wanted, uint8_t non_paced)
14732 {
14733         uint64_t lentim, fill_bw;
14734
14735         /* Lets first see if we are full, if so continue with normal rate */
14736         rack->r_via_fill_cw = 0;
14737         if (ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) > rack->r_ctl.cwnd_to_use)
14738                 return (slot);
14739         if ((ctf_outstanding(rack->rc_tp) + (segsiz-1)) > rack->rc_tp->snd_wnd)
14740                 return (slot);
14741         if (rack->r_ctl.rc_last_us_rtt == 0)
14742                 return (slot);
14743         if (rack->rc_pace_fill_if_rttin_range &&
14744             (rack->r_ctl.rc_last_us_rtt >=
14745              (get_filter_value_small(&rack->r_ctl.rc_gp_min_rtt) * rack->rtt_limit_mul))) {
14746                 /* The rtt is huge, N * smallest, lets not fill */
14747                 return (slot);
14748         }
14749         /*
14750          * first lets calculate the b/w based on the last us-rtt
14751          * and the sndwnd.
14752          */
14753         fill_bw = rack->r_ctl.cwnd_to_use;
14754         /* Take the rwnd if its smaller */
14755         if (fill_bw > rack->rc_tp->snd_wnd)
14756                 fill_bw = rack->rc_tp->snd_wnd;
14757         if (rack->r_fill_less_agg) {
14758                 /*
14759                  * Now take away the inflight (this will reduce our
14760                  * aggressiveness and yeah, if we get that much out in 1RTT
14761                  * we will have had acks come back and still be behind).
14762                  */
14763                 fill_bw -= ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
14764         }
14765         /* Now lets make it into a b/w */
14766         fill_bw *= (uint64_t)HPTS_USEC_IN_SEC;
14767         fill_bw /= (uint64_t)rack->r_ctl.rc_last_us_rtt;
14768         /* We are below the min b/w */
14769         if (non_paced)
14770                 *rate_wanted = fill_bw;
14771         if ((fill_bw < RACK_MIN_BW) || (fill_bw < *rate_wanted))
14772                 return (slot);
14773         if (rack->r_ctl.bw_rate_cap && (fill_bw > rack->r_ctl.bw_rate_cap))
14774                 fill_bw = rack->r_ctl.bw_rate_cap;
14775         rack->r_via_fill_cw = 1;
14776         if (rack->r_rack_hw_rate_caps &&
14777             (rack->r_ctl.crte != NULL)) {
14778                 uint64_t high_rate;
14779
14780                 high_rate = tcp_hw_highest_rate(rack->r_ctl.crte);
14781                 if (fill_bw > high_rate) {
14782                         /* We are capping bw at the highest rate table entry */
14783                         if (*rate_wanted > high_rate) {
14784                                 /* The original rate was also capped */
14785                                 rack->r_via_fill_cw = 0;
14786                         }
14787                         rack_log_hdwr_pacing(rack,
14788                                              fill_bw, high_rate, __LINE__,
14789                                              0, 3);
14790                         fill_bw = high_rate;
14791                         if (capped)
14792                                 *capped = 1;
14793                 }
14794         } else if ((rack->r_ctl.crte == NULL) &&
14795                    (rack->rack_hdrw_pacing == 0) &&
14796                    (rack->rack_hdw_pace_ena) &&
14797                    rack->r_rack_hw_rate_caps &&
14798                    (rack->rack_attempt_hdwr_pace == 0) &&
14799                    (rack->rc_inp->inp_route.ro_nh != NULL) &&
14800                    (rack->rc_inp->inp_route.ro_nh->nh_ifp != NULL)) {
14801                 /*
14802                  * Ok we may have a first attempt that is greater than our top rate
14803                  * lets check.
14804                  */
14805                 uint64_t high_rate;
14806
14807                 high_rate = tcp_hw_highest_rate_ifp(rack->rc_inp->inp_route.ro_nh->nh_ifp, rack->rc_inp);
14808                 if (high_rate) {
14809                         if (fill_bw > high_rate) {
14810                                 fill_bw = high_rate;
14811                                 if (capped)
14812                                         *capped = 1;
14813                         }
14814                 }
14815         }
14816         /*
14817          * Ok fill_bw holds our mythical b/w to fill the cwnd
14818          * in a rtt, what does that time wise equate too?
14819          */
14820         lentim = (uint64_t)(len) * (uint64_t)HPTS_USEC_IN_SEC;
14821         lentim /= fill_bw;
14822         *rate_wanted = fill_bw;
14823         if (non_paced || (lentim < slot)) {
14824                 rack_log_pacing_delay_calc(rack, len, slot, fill_bw,
14825                                            0, lentim, 12, __LINE__, NULL, 0);
14826                 return ((int32_t)lentim);
14827         } else
14828                 return (slot);
14829 }
14830
14831 static int32_t
14832 rack_get_pacing_delay(struct tcp_rack *rack, struct tcpcb *tp, uint32_t len, struct rack_sendmap *rsm, uint32_t segsiz)
14833 {
14834         uint64_t srtt;
14835         int32_t slot = 0;
14836         int can_start_hw_pacing = 1;
14837         int err;
14838
14839         if (rack->rc_always_pace == 0) {
14840                 /*
14841                  * We use the most optimistic possible cwnd/srtt for
14842                  * sending calculations. This will make our
14843                  * calculation anticipate getting more through
14844                  * quicker then possible. But thats ok we don't want
14845                  * the peer to have a gap in data sending.
14846                  */
14847                 uint64_t cwnd, tr_perms = 0;
14848                 int32_t reduce = 0;
14849
14850         old_method:
14851                 /*
14852                  * We keep no precise pacing with the old method
14853                  * instead we use the pacer to mitigate bursts.
14854                  */
14855                 if (rack->r_ctl.rc_rack_min_rtt)
14856                         srtt = rack->r_ctl.rc_rack_min_rtt;
14857                 else
14858                         srtt = max(tp->t_srtt, 1);
14859                 if (rack->r_ctl.rc_rack_largest_cwnd)
14860                         cwnd = rack->r_ctl.rc_rack_largest_cwnd;
14861                 else
14862                         cwnd = rack->r_ctl.cwnd_to_use;
14863                 /* Inflate cwnd by 1000 so srtt of usecs is in ms */
14864                 tr_perms = (cwnd * 1000) / srtt;
14865                 if (tr_perms == 0) {
14866                         tr_perms = ctf_fixed_maxseg(tp);
14867                 }
14868                 /*
14869                  * Calculate how long this will take to drain, if
14870                  * the calculation comes out to zero, thats ok we
14871                  * will use send_a_lot to possibly spin around for
14872                  * more increasing tot_len_this_send to the point
14873                  * that its going to require a pace, or we hit the
14874                  * cwnd. Which in that case we are just waiting for
14875                  * a ACK.
14876                  */
14877                 slot = len / tr_perms;
14878                 /* Now do we reduce the time so we don't run dry? */
14879                 if (slot && rack_slot_reduction) {
14880                         reduce = (slot / rack_slot_reduction);
14881                         if (reduce < slot) {
14882                                 slot -= reduce;
14883                         } else
14884                                 slot = 0;
14885                 }
14886                 slot *= HPTS_USEC_IN_MSEC;
14887                 if (rack->rc_pace_to_cwnd) {
14888                         uint64_t rate_wanted = 0;
14889
14890                         slot = pace_to_fill_cwnd(rack, slot, len, segsiz, NULL, &rate_wanted, 1);
14891                         rack->rc_ack_can_sendout_data = 1;
14892                         rack_log_pacing_delay_calc(rack, len, slot, rate_wanted, 0, 0, 14, __LINE__, NULL, 0);
14893                 } else
14894                         rack_log_pacing_delay_calc(rack, len, slot, tr_perms, reduce, 0, 7, __LINE__, NULL, 0);
14895         } else {
14896                 uint64_t bw_est, res, lentim, rate_wanted;
14897                 uint32_t orig_val, segs, oh;
14898                 int capped = 0;
14899                 int prev_fill;
14900
14901                 if ((rack->r_rr_config == 1) && rsm) {
14902                         return (rack->r_ctl.rc_min_to);
14903                 }
14904                 if (rack->use_fixed_rate) {
14905                         rate_wanted = bw_est = rack_get_fixed_pacing_bw(rack);
14906                 } else if ((rack->r_ctl.init_rate == 0) &&
14907 #ifdef NETFLIX_PEAKRATE
14908                            (rack->rc_tp->t_maxpeakrate == 0) &&
14909 #endif
14910                            (rack->r_ctl.gp_bw == 0)) {
14911                         /* no way to yet do an estimate */
14912                         bw_est = rate_wanted = 0;
14913                 } else {
14914                         bw_est = rack_get_bw(rack);
14915                         rate_wanted = rack_get_output_bw(rack, bw_est, rsm, &capped);
14916                 }
14917                 if ((bw_est == 0) || (rate_wanted == 0) ||
14918                     ((rack->gp_ready == 0) && (rack->use_fixed_rate == 0))) {
14919                         /*
14920                          * No way yet to make a b/w estimate or
14921                          * our raise is set incorrectly.
14922                          */
14923                         goto old_method;
14924                 }
14925                 /* We need to account for all the overheads */
14926                 segs = (len + segsiz - 1) / segsiz;
14927                 /*
14928                  * We need the diff between 1514 bytes (e-mtu with e-hdr)
14929                  * and how much data we put in each packet. Yes this
14930                  * means we may be off if we are larger than 1500 bytes
14931                  * or smaller. But this just makes us more conservative.
14932                  */
14933                 if (rack_hw_rate_min &&
14934                     (bw_est < rack_hw_rate_min))
14935                         can_start_hw_pacing = 0;
14936                 if (ETHERNET_SEGMENT_SIZE > segsiz)
14937                         oh = ETHERNET_SEGMENT_SIZE - segsiz;
14938                 else
14939                         oh = 0;
14940                 segs *= oh;
14941                 lentim = (uint64_t)(len + segs) * (uint64_t)HPTS_USEC_IN_SEC;
14942                 res = lentim / rate_wanted;
14943                 slot = (uint32_t)res;
14944                 orig_val = rack->r_ctl.rc_pace_max_segs;
14945                 if (rack->r_ctl.crte == NULL) {
14946                         /*
14947                          * Only do this if we are not hardware pacing
14948                          * since if we are doing hw-pacing below we will
14949                          * set make a call after setting up or changing
14950                          * the rate.
14951                          */
14952                         rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
14953                 } else if (rack->rc_inp->inp_snd_tag == NULL) {
14954                         /*
14955                          * We lost our rate somehow, this can happen
14956                          * if the interface changed underneath us.
14957                          */
14958                         tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp);
14959                         rack->r_ctl.crte = NULL;
14960                         /* Lets re-allow attempting to setup pacing */
14961                         rack->rack_hdrw_pacing = 0;
14962                         rack->rack_attempt_hdwr_pace = 0;
14963                         rack_log_hdwr_pacing(rack,
14964                                              rate_wanted, bw_est, __LINE__,
14965                                              0, 6);
14966                 }
14967                 /* Did we change the TSO size, if so log it */
14968                 if (rack->r_ctl.rc_pace_max_segs != orig_val)
14969                         rack_log_pacing_delay_calc(rack, len, slot, orig_val, 0, 0, 15, __LINE__, NULL, 0);
14970                 prev_fill = rack->r_via_fill_cw;
14971                 if ((rack->rc_pace_to_cwnd) &&
14972                     (capped == 0) &&
14973                     (rack->use_fixed_rate == 0) &&
14974                     (rack->in_probe_rtt == 0) &&
14975                     (IN_FASTRECOVERY(rack->rc_tp->t_flags) == 0)) {
14976                         /*
14977                          * We want to pace at our rate *or* faster to
14978                          * fill the cwnd to the max if its not full.
14979                          */
14980                         slot = pace_to_fill_cwnd(rack, slot, (len+segs), segsiz, &capped, &rate_wanted, 0);
14981                 }
14982                 if ((rack->rc_inp->inp_route.ro_nh != NULL) &&
14983                     (rack->rc_inp->inp_route.ro_nh->nh_ifp != NULL)) {
14984                         if ((rack->rack_hdw_pace_ena) &&
14985                             (can_start_hw_pacing > 0) &&
14986                             (rack->rack_hdrw_pacing == 0) &&
14987                             (rack->rack_attempt_hdwr_pace == 0)) {
14988                                 /*
14989                                  * Lets attempt to turn on hardware pacing
14990                                  * if we can.
14991                                  */
14992                                 rack->rack_attempt_hdwr_pace = 1;
14993                                 rack->r_ctl.crte = tcp_set_pacing_rate(rack->rc_tp,
14994                                                                        rack->rc_inp->inp_route.ro_nh->nh_ifp,
14995                                                                        rate_wanted,
14996                                                                        RS_PACING_GEQ,
14997                                                                        &err, &rack->r_ctl.crte_prev_rate);
14998                                 if (rack->r_ctl.crte) {
14999                                         rack->rack_hdrw_pacing = 1;
15000                                         rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size(tp, rate_wanted, segsiz,
15001                                                                                                  0, rack->r_ctl.crte,
15002                                                                                                  NULL);
15003                                         rack_log_hdwr_pacing(rack,
15004                                                              rate_wanted, rack->r_ctl.crte->rate, __LINE__,
15005                                                              err, 0);
15006                                         rack->r_ctl.last_hw_bw_req = rate_wanted;
15007                                 } else {
15008                                         counter_u64_add(rack_hw_pace_init_fail, 1);
15009                                 }
15010                         } else if (rack->rack_hdrw_pacing &&
15011                                    (rack->r_ctl.last_hw_bw_req != rate_wanted)) {
15012                                 /* Do we need to adjust our rate? */
15013                                 const struct tcp_hwrate_limit_table *nrte;
15014
15015                                 if (rack->r_up_only &&
15016                                     (rate_wanted < rack->r_ctl.crte->rate)) {
15017                                         /**
15018                                          * We have four possible states here
15019                                          * having to do with the previous time
15020                                          * and this time.
15021                                          *   previous  |  this-time
15022                                          * A)     0      |     0   -- fill_cw not in the picture
15023                                          * B)     1      |     0   -- we were doing a fill-cw but now are not
15024                                          * C)     1      |     1   -- all rates from fill_cw
15025                                          * D)     0      |     1   -- we were doing non-fill and now we are filling
15026                                          *
15027                                          * For case A, C and D we don't allow a drop. But for
15028                                          * case B where we now our on our steady rate we do
15029                                          * allow a drop.
15030                                          *
15031                                          */
15032                                         if (!((prev_fill == 1) && (rack->r_via_fill_cw == 0)))
15033                                                 goto done_w_hdwr;
15034                                 }
15035                                 if ((rate_wanted > rack->r_ctl.crte->rate) ||
15036                                     (rate_wanted <= rack->r_ctl.crte_prev_rate)) {
15037                                         if (rack_hw_rate_to_low &&
15038                                             (bw_est < rack_hw_rate_to_low)) {
15039                                                 /*
15040                                                  * The pacing rate is too low for hardware, but
15041                                                  * do allow hardware pacing to be restarted.
15042                                                  */
15043                                                 rack_log_hdwr_pacing(rack,
15044                                                              bw_est, rack->r_ctl.crte->rate, __LINE__,
15045                                                              0, 5);
15046                                                 tcp_rel_pacing_rate(rack->r_ctl.crte, rack->rc_tp);
15047                                                 rack->r_ctl.crte = NULL;
15048                                                 rack->rack_attempt_hdwr_pace = 0;
15049                                                 rack->rack_hdrw_pacing = 0;
15050                                                 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, &rate_wanted);
15051                                                 goto done_w_hdwr;
15052                                         }
15053                                         nrte = tcp_chg_pacing_rate(rack->r_ctl.crte,
15054                                                                    rack->rc_tp,
15055                                                                    rack->rc_inp->inp_route.ro_nh->nh_ifp,
15056                                                                    rate_wanted,
15057                                                                    RS_PACING_GEQ,
15058                                                                    &err, &rack->r_ctl.crte_prev_rate);
15059                                         if (nrte == NULL) {
15060                                                 /* Lost the rate */
15061                                                 rack->rack_hdrw_pacing = 0;
15062                                                 rack->r_ctl.crte = NULL;
15063                                                 rack_log_hdwr_pacing(rack,
15064                                                                      rate_wanted, 0, __LINE__,
15065                                                                      err, 1);
15066                                                 rack_set_pace_segments(rack->rc_tp, rack, __LINE__, &rate_wanted);
15067                                                 counter_u64_add(rack_hw_pace_lost, 1);
15068                                         } else if (nrte != rack->r_ctl.crte) {
15069                                                 rack->r_ctl.crte = nrte;
15070                                                 rack->r_ctl.rc_pace_max_segs = tcp_get_pacing_burst_size(tp, rate_wanted,
15071                                                                                                          segsiz, 0,
15072                                                                                                          rack->r_ctl.crte,
15073                                                                                                          NULL);
15074                                                 rack_log_hdwr_pacing(rack,
15075                                                                      rate_wanted, rack->r_ctl.crte->rate, __LINE__,
15076                                                                      err, 2);
15077                                                 rack->r_ctl.last_hw_bw_req = rate_wanted;
15078                                         }
15079                                 } else {
15080                                         /* We just need to adjust the segment size */
15081                                         rack_set_pace_segments(rack->rc_tp, rack, __LINE__, &rate_wanted);
15082                                         rack_log_hdwr_pacing(rack,
15083                                                              rate_wanted, rack->r_ctl.crte->rate, __LINE__,
15084                                                              0, 4);
15085                                         rack->r_ctl.last_hw_bw_req = rate_wanted;
15086                                 }
15087                         }
15088                 }
15089                 if ((rack->r_ctl.crte != NULL) &&
15090                     (rack->r_ctl.crte->rate == rate_wanted)) {
15091                         /*
15092                          * We need to add a extra if the rates
15093                          * are exactly matched. The idea is
15094                          * we want the software to make sure the
15095                          * queue is empty before adding more, this
15096                          * gives us N MSS extra pace times where
15097                          * N is our sysctl
15098                          */
15099                         slot += (rack->r_ctl.crte->time_between * rack_hw_pace_extra_slots);
15100                 }
15101 done_w_hdwr:
15102                 if (rack_limit_time_with_srtt &&
15103                     (rack->use_fixed_rate == 0) &&
15104 #ifdef NETFLIX_PEAKRATE
15105                     (rack->rc_tp->t_maxpeakrate == 0) &&
15106 #endif
15107                     (rack->rack_hdrw_pacing == 0)) {
15108                         /*
15109                          * Sanity check, we do not allow the pacing delay
15110                          * to be longer than the SRTT of the path. If it is
15111                          * a slow path, then adding a packet should increase
15112                          * the RTT and compensate for this i.e. the srtt will
15113                          * be greater so the allowed pacing time will be greater.
15114                          *
15115                          * Note this restriction is not for where a peak rate
15116                          * is set, we are doing fixed pacing or hardware pacing.
15117                          */
15118                         if (rack->rc_tp->t_srtt)
15119                                 srtt = rack->rc_tp->t_srtt;
15120                         else
15121                                 srtt = RACK_INITIAL_RTO * HPTS_USEC_IN_MSEC;    /* its in ms convert */
15122                         if (srtt < (uint64_t)slot) {
15123                                 rack_log_pacing_delay_calc(rack, srtt, slot, rate_wanted, bw_est, lentim, 99, __LINE__, NULL, 0);
15124                                 slot = srtt;
15125                         }
15126                 }
15127                 rack_log_pacing_delay_calc(rack, len, slot, rate_wanted, bw_est, lentim, 2, __LINE__, rsm, 0);
15128         }
15129         if (rack->r_ctl.crte && (rack->r_ctl.crte->rs_num_enobufs > 0)) {
15130                 /*
15131                  * If this rate is seeing enobufs when it
15132                  * goes to send then either the nic is out
15133                  * of gas or we are mis-estimating the time
15134                  * somehow and not letting the queue empty
15135                  * completely. Lets add to the pacing time.
15136                  */
15137                 int hw_boost_delay;
15138
15139                 hw_boost_delay = rack->r_ctl.crte->time_between * rack_enobuf_hw_boost_mult;
15140                 if (hw_boost_delay > rack_enobuf_hw_max)
15141                         hw_boost_delay = rack_enobuf_hw_max;
15142                 else if (hw_boost_delay < rack_enobuf_hw_min)
15143                         hw_boost_delay = rack_enobuf_hw_min;
15144                 slot += hw_boost_delay;
15145         }
15146         return (slot);
15147 }
15148
15149 static void
15150 rack_start_gp_measurement(struct tcpcb *tp, struct tcp_rack *rack,
15151     tcp_seq startseq, uint32_t sb_offset)
15152 {
15153         struct rack_sendmap *my_rsm = NULL;
15154         struct rack_sendmap fe;
15155
15156         if (tp->t_state < TCPS_ESTABLISHED) {
15157                 /*
15158                  * We don't start any measurements if we are
15159                  * not at least established.
15160                  */
15161                 return;
15162         }
15163         if (tp->t_state >= TCPS_FIN_WAIT_1) {
15164                 /*
15165                  * We will get no more data into the SB
15166                  * this means we need to have the data available
15167                  * before we start a measurement.
15168                  */
15169
15170                 if (sbavail(&tp->t_inpcb->inp_socket->so_snd) <
15171                     max(rc_init_window(rack),
15172                         (MIN_GP_WIN * ctf_fixed_maxseg(tp)))) {
15173                         /* Nope not enough data */
15174                         return;
15175                 }
15176         }
15177         tp->t_flags |= TF_GPUTINPROG;
15178         rack->r_ctl.rc_gp_lowrtt = 0xffffffff;
15179         rack->r_ctl.rc_gp_high_rwnd = rack->rc_tp->snd_wnd;
15180         tp->gput_seq = startseq;
15181         rack->app_limited_needs_set = 0;
15182         if (rack->in_probe_rtt)
15183                 rack->measure_saw_probe_rtt = 1;
15184         else if ((rack->measure_saw_probe_rtt) &&
15185                  (SEQ_GEQ(tp->gput_seq, rack->r_ctl.rc_probertt_sndmax_atexit)))
15186                 rack->measure_saw_probe_rtt = 0;
15187         if (rack->rc_gp_filled)
15188                 tp->gput_ts = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time);
15189         else {
15190                 /* Special case initial measurement */
15191                 struct timeval tv;
15192
15193                 tp->gput_ts = tcp_get_usecs(&tv);
15194                 rack->r_ctl.rc_gp_output_ts = rack_to_usec_ts(&tv);
15195         }
15196         /*
15197          * We take a guess out into the future,
15198          * if we have no measurement and no
15199          * initial rate, we measure the first
15200          * initial-windows worth of data to
15201          * speed up getting some GP measurement and
15202          * thus start pacing.
15203          */
15204         if ((rack->rc_gp_filled == 0) && (rack->r_ctl.init_rate == 0)) {
15205                 rack->app_limited_needs_set = 1;
15206                 tp->gput_ack = startseq + max(rc_init_window(rack),
15207                                               (MIN_GP_WIN * ctf_fixed_maxseg(tp)));
15208                 rack_log_pacing_delay_calc(rack,
15209                                            tp->gput_seq,
15210                                            tp->gput_ack,
15211                                            0,
15212                                            tp->gput_ts,
15213                                            rack->r_ctl.rc_app_limited_cnt,
15214                                            9,
15215                                            __LINE__, NULL, 0);
15216                 return;
15217         }
15218         if (sb_offset) {
15219                 /*
15220                  * We are out somewhere in the sb
15221                  * can we use the already outstanding data?
15222                  */
15223                 if (rack->r_ctl.rc_app_limited_cnt == 0) {
15224                         /*
15225                          * Yes first one is good and in this case
15226                          * the tp->gput_ts is correctly set based on
15227                          * the last ack that arrived (no need to
15228                          * set things up when an ack comes in).
15229                          */
15230                         my_rsm = RB_MIN(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
15231                         if ((my_rsm == NULL) ||
15232                             (my_rsm->r_rtr_cnt != 1)) {
15233                                 /* retransmission? */
15234                                 goto use_latest;
15235                         }
15236                 } else {
15237                         if (rack->r_ctl.rc_first_appl == NULL) {
15238                                 /*
15239                                  * If rc_first_appl is NULL
15240                                  * then the cnt should be 0.
15241                                  * This is probably an error, maybe
15242                                  * a KASSERT would be approprate.
15243                                  */
15244                                 goto use_latest;
15245                         }
15246                         /*
15247                          * If we have a marker pointer to the last one that is
15248                          * app limited we can use that, but we need to set
15249                          * things up so that when it gets ack'ed we record
15250                          * the ack time (if its not already acked).
15251                          */
15252                         rack->app_limited_needs_set = 1;
15253                         /*
15254                          * We want to get to the rsm that is either
15255                          * next with space i.e. over 1 MSS or the one
15256                          * after that (after the app-limited).
15257                          */
15258                         my_rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree,
15259                                          rack->r_ctl.rc_first_appl);
15260                         if (my_rsm) {
15261                                 if ((my_rsm->r_end - my_rsm->r_start) <= ctf_fixed_maxseg(tp))
15262                                         /* Have to use the next one */
15263                                         my_rsm = RB_NEXT(rack_rb_tree_head, &rack->r_ctl.rc_mtree,
15264                                                          my_rsm);
15265                                 else {
15266                                         /* Use after the first MSS of it is acked */
15267                                         tp->gput_seq = my_rsm->r_start + ctf_fixed_maxseg(tp);
15268                                         goto start_set;
15269                                 }
15270                         }
15271                         if ((my_rsm == NULL) ||
15272                             (my_rsm->r_rtr_cnt != 1)) {
15273                                 /*
15274                                  * Either its a retransmit or
15275                                  * the last is the app-limited one.
15276                                  */
15277                                 goto use_latest;
15278                         }
15279                 }
15280                 tp->gput_seq = my_rsm->r_start;
15281 start_set:
15282                 if (my_rsm->r_flags & RACK_ACKED) {
15283                         /*
15284                          * This one has been acked use the arrival ack time
15285                          */
15286                         tp->gput_ts = (uint32_t)my_rsm->r_ack_arrival;
15287                         rack->app_limited_needs_set = 0;
15288                 }
15289                 rack->r_ctl.rc_gp_output_ts = my_rsm->r_tim_lastsent[(my_rsm->r_rtr_cnt-1)];
15290                 tp->gput_ack = tp->gput_seq + rack_get_measure_window(tp, rack);
15291                 rack_log_pacing_delay_calc(rack,
15292                                            tp->gput_seq,
15293                                            tp->gput_ack,
15294                                            (uint64_t)my_rsm,
15295                                            tp->gput_ts,
15296                                            rack->r_ctl.rc_app_limited_cnt,
15297                                            9,
15298                                            __LINE__, NULL, 0);
15299                 return;
15300         }
15301
15302 use_latest:
15303         /*
15304          * We don't know how long we may have been
15305          * idle or if this is the first-send. Lets
15306          * setup the flag so we will trim off
15307          * the first ack'd data so we get a true
15308          * measurement.
15309          */
15310         rack->app_limited_needs_set = 1;
15311         tp->gput_ack = startseq + rack_get_measure_window(tp, rack);
15312         /* Find this guy so we can pull the send time */
15313         fe.r_start = startseq;
15314         my_rsm = RB_FIND(rack_rb_tree_head, &rack->r_ctl.rc_mtree, &fe);
15315         if (my_rsm) {
15316                 rack->r_ctl.rc_gp_output_ts = my_rsm->r_tim_lastsent[(my_rsm->r_rtr_cnt-1)];
15317                 if (my_rsm->r_flags & RACK_ACKED) {
15318                         /*
15319                          * Unlikely since its probably what was
15320                          * just transmitted (but I am paranoid).
15321                          */
15322                         tp->gput_ts = (uint32_t)my_rsm->r_ack_arrival;
15323                         rack->app_limited_needs_set = 0;
15324                 }
15325                 if (SEQ_LT(my_rsm->r_start, tp->gput_seq)) {
15326                         /* This also is unlikely */
15327                         tp->gput_seq = my_rsm->r_start;
15328                 }
15329         } else {
15330                 /*
15331                  * TSNH unless we have some send-map limit,
15332                  * and even at that it should not be hitting
15333                  * that limit (we should have stopped sending).
15334                  */
15335                 struct timeval tv;
15336
15337                 microuptime(&tv);
15338                 rack->r_ctl.rc_gp_output_ts = rack_to_usec_ts(&tv);
15339         }
15340         rack_log_pacing_delay_calc(rack,
15341                                    tp->gput_seq,
15342                                    tp->gput_ack,
15343                                    (uint64_t)my_rsm,
15344                                    tp->gput_ts,
15345                                    rack->r_ctl.rc_app_limited_cnt,
15346                                    9, __LINE__, NULL, 0);
15347 }
15348
15349 static inline uint32_t
15350 rack_what_can_we_send(struct tcpcb *tp, struct tcp_rack *rack,  uint32_t cwnd_to_use,
15351     uint32_t avail, int32_t sb_offset)
15352 {
15353         uint32_t len;
15354         uint32_t sendwin;
15355
15356         if (tp->snd_wnd > cwnd_to_use)
15357                 sendwin = cwnd_to_use;
15358         else
15359                 sendwin = tp->snd_wnd;
15360         if (ctf_outstanding(tp) >= tp->snd_wnd) {
15361                 /* We never want to go over our peers rcv-window */
15362                 len = 0;
15363         } else {
15364                 uint32_t flight;
15365
15366                 flight = ctf_flight_size(tp, rack->r_ctl.rc_sacked);
15367                 if (flight >= sendwin) {
15368                         /*
15369                          * We have in flight what we are allowed by cwnd (if
15370                          * it was rwnd blocking it would have hit above out
15371                          * >= tp->snd_wnd).
15372                          */
15373                         return (0);
15374                 }
15375                 len = sendwin - flight;
15376                 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) {
15377                         /* We would send too much (beyond the rwnd) */
15378                         len = tp->snd_wnd - ctf_outstanding(tp);
15379                 }
15380                 if ((len + sb_offset) > avail) {
15381                         /*
15382                          * We don't have that much in the SB, how much is
15383                          * there?
15384                          */
15385                         len = avail - sb_offset;
15386                 }
15387         }
15388         return (len);
15389 }
15390
15391 static void
15392 rack_log_fsb(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_t flags,
15393              unsigned ipoptlen, int32_t orig_len, int32_t len, int error,
15394              int rsm_is_null, int optlen, int line, uint16_t mode)
15395 {
15396         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
15397                 union tcp_log_stackspecific log;
15398                 struct timeval tv;
15399
15400                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
15401                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
15402                 log.u_bbr.flex1 = error;
15403                 log.u_bbr.flex2 = flags;
15404                 log.u_bbr.flex3 = rsm_is_null;
15405                 log.u_bbr.flex4 = ipoptlen;
15406                 log.u_bbr.flex5 = tp->rcv_numsacks;
15407                 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early;
15408                 log.u_bbr.flex7 = optlen;
15409                 log.u_bbr.flex8 = rack->r_fsb_inited;
15410                 log.u_bbr.applimited = rack->r_fast_output;
15411                 log.u_bbr.bw_inuse = rack_get_bw(rack);
15412                 log.u_bbr.pacing_gain = rack_get_output_gain(rack, NULL);
15413                 log.u_bbr.cwnd_gain = mode;
15414                 log.u_bbr.pkts_out = orig_len;
15415                 log.u_bbr.lt_epoch = len;
15416                 log.u_bbr.delivered = line;
15417                 log.u_bbr.timeStamp = tcp_get_usecs(&tv);
15418                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
15419                 tcp_log_event_(tp, NULL, &so->so_rcv, &so->so_snd, TCP_LOG_FSB, 0,
15420                                len, &log, false, NULL, NULL, 0, &tv);
15421         }
15422 }
15423
15424
15425 static struct mbuf *
15426 rack_fo_base_copym(struct mbuf *the_m, uint32_t the_off, int32_t *plen,
15427                    struct rack_fast_send_blk *fsb,
15428                    int32_t seglimit, int32_t segsize, int hw_tls)
15429 {
15430 #ifdef KERN_TLS
15431         struct ktls_session *tls, *ntls;
15432 #ifdef INVARIANTS
15433         struct mbuf *start;
15434 #endif
15435 #endif
15436         struct mbuf *m, *n, **np, *smb;
15437         struct mbuf *top;
15438         int32_t off, soff;
15439         int32_t len = *plen;
15440         int32_t fragsize;
15441         int32_t len_cp = 0;
15442         uint32_t mlen, frags;
15443
15444         soff = off = the_off;
15445         smb = m = the_m;
15446         np = &top;
15447         top = NULL;
15448 #ifdef KERN_TLS
15449         if (hw_tls && (m->m_flags & M_EXTPG))
15450                 tls = m->m_epg_tls;
15451         else
15452                 tls = NULL;
15453 #ifdef INVARIANTS
15454         start = m;
15455 #endif
15456 #endif
15457         while (len > 0) {
15458                 if (m == NULL) {
15459                         *plen = len_cp;
15460                         break;
15461                 }
15462 #ifdef KERN_TLS
15463                 if (hw_tls) {
15464                         if (m->m_flags & M_EXTPG)
15465                                 ntls = m->m_epg_tls;
15466                         else
15467                                 ntls = NULL;
15468
15469                         /*
15470                          * Avoid mixing TLS records with handshake
15471                          * data or TLS records from different
15472                          * sessions.
15473                          */
15474                         if (tls != ntls) {
15475                                 MPASS(m != start);
15476                                 *plen = len_cp;
15477                                 break;
15478                         }
15479                 }
15480 #endif
15481                 mlen = min(len, m->m_len - off);
15482                 if (seglimit) {
15483                         /*
15484                          * For M_EXTPG mbufs, add 3 segments
15485                          * + 1 in case we are crossing page boundaries
15486                          * + 2 in case the TLS hdr/trailer are used
15487                          * It is cheaper to just add the segments
15488                          * than it is to take the cache miss to look
15489                          * at the mbuf ext_pgs state in detail.
15490                          */
15491                         if (m->m_flags & M_EXTPG) {
15492                                 fragsize = min(segsize, PAGE_SIZE);
15493                                 frags = 3;
15494                         } else {
15495                                 fragsize = segsize;
15496                                 frags = 0;
15497                         }
15498
15499                         /* Break if we really can't fit anymore. */
15500                         if ((frags + 1) >= seglimit) {
15501                                 *plen = len_cp;
15502                                 break;
15503                         }
15504
15505                         /*
15506                          * Reduce size if you can't copy the whole
15507                          * mbuf. If we can't copy the whole mbuf, also
15508                          * adjust len so the loop will end after this
15509                          * mbuf.
15510                          */
15511                         if ((frags + howmany(mlen, fragsize)) >= seglimit) {
15512                                 mlen = (seglimit - frags - 1) * fragsize;
15513                                 len = mlen;
15514                                 *plen = len_cp + len;
15515                         }
15516                         frags += howmany(mlen, fragsize);
15517                         if (frags == 0)
15518                                 frags++;
15519                         seglimit -= frags;
15520                         KASSERT(seglimit > 0,
15521                             ("%s: seglimit went too low", __func__));
15522                 }
15523                 n = m_get(M_NOWAIT, m->m_type);
15524                 *np = n;
15525                 if (n == NULL)
15526                         goto nospace;
15527                 n->m_len = mlen;
15528                 soff += mlen;
15529                 len_cp += n->m_len;
15530                 if (m->m_flags & (M_EXT|M_EXTPG)) {
15531                         n->m_data = m->m_data + off;
15532                         mb_dupcl(n, m);
15533                 } else {
15534                         bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
15535                             (u_int)n->m_len);
15536                 }
15537                 len -= n->m_len;
15538                 off = 0;
15539                 m = m->m_next;
15540                 np = &n->m_next;
15541                 if (len || (soff == smb->m_len)) {
15542                         /*
15543                          * We have more so we move forward  or
15544                          * we have consumed the entire mbuf and
15545                          * len has fell to 0.
15546                          */
15547                         soff = 0;
15548                         smb = m;
15549                 }
15550
15551         }
15552         if (fsb != NULL) {
15553                 fsb->m = smb;
15554                 fsb->off = soff;
15555                 if (smb) {
15556                         /*
15557                          * Save off the size of the mbuf. We do
15558                          * this so that we can recognize when it
15559                          * has been trimmed by sbcut() as acks
15560                          * come in.
15561                          */
15562                         fsb->o_m_len = smb->m_len;
15563                 } else {
15564                         /*
15565                          * This is the case where the next mbuf went to NULL. This
15566                          * means with this copy we have sent everything in the sb.
15567                          * In theory we could clear the fast_output flag, but lets
15568                          * not since its possible that we could get more added
15569                          * and acks that call the extend function which would let
15570                          * us send more.
15571                          */
15572                         fsb->o_m_len = 0;
15573                 }
15574         }
15575         return (top);
15576 nospace:
15577         if (top)
15578                 m_freem(top);
15579         return (NULL);
15580
15581 }
15582
15583 /*
15584  * This is a copy of m_copym(), taking the TSO segment size/limit
15585  * constraints into account, and advancing the sndptr as it goes.
15586  */
15587 static struct mbuf *
15588 rack_fo_m_copym(struct tcp_rack *rack, int32_t *plen,
15589                 int32_t seglimit, int32_t segsize, struct mbuf **s_mb, int *s_soff)
15590 {
15591         struct mbuf *m, *n;
15592         int32_t soff;
15593
15594         soff = rack->r_ctl.fsb.off;
15595         m = rack->r_ctl.fsb.m;
15596         if (rack->r_ctl.fsb.o_m_len > m->m_len) {
15597                 /*
15598                  * The mbuf had the front of it chopped off by an ack
15599                  * we need to adjust the soff/off by that difference.
15600                  */
15601                 uint32_t delta;
15602
15603                 delta = rack->r_ctl.fsb.o_m_len - m->m_len;
15604                 soff -= delta;
15605         } else if (rack->r_ctl.fsb.o_m_len < m->m_len) {
15606                 /*
15607                  * The mbuf was expanded probably by
15608                  * a m_compress. Just update o_m_len.
15609                  */
15610                 rack->r_ctl.fsb.o_m_len = m->m_len;
15611         }
15612         KASSERT(soff >= 0, ("%s, negative off %d", __FUNCTION__, soff));
15613         KASSERT(*plen >= 0, ("%s, negative len %d", __FUNCTION__, *plen));
15614         KASSERT(soff < m->m_len, ("%s rack:%p len:%u m:%p m->m_len:%u < off?",
15615                                  __FUNCTION__,
15616                                  rack, *plen, m, m->m_len));
15617         /* Save off the right location before we copy and advance */
15618         *s_soff = soff;
15619         *s_mb = rack->r_ctl.fsb.m;
15620         n = rack_fo_base_copym(m, soff, plen,
15621                                &rack->r_ctl.fsb,
15622                                seglimit, segsize, rack->r_ctl.fsb.hw_tls);
15623         return (n);
15624 }
15625
15626 static int
15627 rack_fast_rsm_output(struct tcpcb *tp, struct tcp_rack *rack, struct rack_sendmap *rsm,
15628                      uint64_t ts_val, uint32_t cts, uint32_t ms_cts, struct timeval *tv, int len, uint8_t doing_tlp)
15629 {
15630         /*
15631          * Enter the fast retransmit path. We are given that a sched_pin is
15632          * in place (if accounting is compliled in) and the cycle count taken
15633          * at the entry is in the ts_val. The concept her is that the rsm
15634          * now holds the mbuf offsets and such so we can directly transmit
15635          * without a lot of overhead, the len field is already set for
15636          * us to prohibit us from sending too much (usually its 1MSS).
15637          */
15638         struct ip *ip = NULL;
15639         struct udphdr *udp = NULL;
15640         struct tcphdr *th = NULL;
15641         struct mbuf *m = NULL;
15642         struct inpcb *inp;
15643         uint8_t *cpto;
15644         struct tcp_log_buffer *lgb;
15645 #ifdef TCP_ACCOUNTING
15646         uint64_t crtsc;
15647         int cnt_thru = 1;
15648 #endif
15649         struct tcpopt to;
15650         u_char opt[TCP_MAXOLEN];
15651         uint32_t hdrlen, optlen;
15652         int32_t slot, segsiz, max_val, tso = 0, error, ulen = 0;
15653         uint16_t flags;
15654         uint32_t if_hw_tsomaxsegcount = 0, startseq;
15655         uint32_t if_hw_tsomaxsegsize;
15656
15657 #ifdef INET6
15658         struct ip6_hdr *ip6 = NULL;
15659
15660         if (rack->r_is_v6) {
15661                 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr;
15662                 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
15663         } else
15664 #endif                          /* INET6 */
15665         {
15666                 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
15667                 hdrlen = sizeof(struct tcpiphdr);
15668         }
15669         if (tp->t_port && (V_tcp_udp_tunneling_port == 0)) {
15670                 goto failed;
15671         }
15672         if (doing_tlp) {
15673                 /* Its a TLP add the flag, it may already be there but be sure */
15674                 rsm->r_flags |= RACK_TLP;
15675         } else {
15676                 /* If it was a TLP it is not not on this retransmit */
15677                 rsm->r_flags &= ~RACK_TLP;
15678         }
15679         startseq = rsm->r_start;
15680         segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
15681         inp = rack->rc_inp;
15682         to.to_flags = 0;
15683         flags = tcp_outflags[tp->t_state];
15684         if (flags & (TH_SYN|TH_RST)) {
15685                 goto failed;
15686         }
15687         if (rsm->r_flags & RACK_HAS_FIN) {
15688                 /* We can't send a FIN here */
15689                 goto failed;
15690         }
15691         if (flags & TH_FIN) {
15692                 /* We never send a FIN */
15693                 flags &= ~TH_FIN;
15694         }
15695         if (tp->t_flags & TF_RCVD_TSTMP) {
15696                 to.to_tsval = ms_cts + tp->ts_offset;
15697                 to.to_tsecr = tp->ts_recent;
15698                 to.to_flags = TOF_TS;
15699         }
15700         optlen = tcp_addoptions(&to, opt);
15701         hdrlen += optlen;
15702         udp = rack->r_ctl.fsb.udp;
15703         if (udp)
15704                 hdrlen += sizeof(struct udphdr);
15705         if (rack->r_ctl.rc_pace_max_segs)
15706                 max_val = rack->r_ctl.rc_pace_max_segs;
15707         else if (rack->rc_user_set_max_segs)
15708                 max_val = rack->rc_user_set_max_segs * segsiz;
15709         else
15710                 max_val = len;
15711         if ((tp->t_flags & TF_TSO) &&
15712             V_tcp_do_tso &&
15713             (len > segsiz) &&
15714             (tp->t_port == 0))
15715                 tso = 1;
15716 #ifdef INET6
15717         if (MHLEN < hdrlen + max_linkhdr)
15718                 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
15719         else
15720 #endif
15721                 m = m_gethdr(M_NOWAIT, MT_DATA);
15722         if (m == NULL)
15723                 goto failed;
15724         m->m_data += max_linkhdr;
15725         m->m_len = hdrlen;
15726         th = rack->r_ctl.fsb.th;
15727         /* Establish the len to send */
15728         if (len > max_val)
15729                 len = max_val;
15730         if ((tso) && (len + optlen > tp->t_maxseg)) {
15731                 uint32_t if_hw_tsomax;
15732                 int32_t max_len;
15733
15734                 /* extract TSO information */
15735                 if_hw_tsomax = tp->t_tsomax;
15736                 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
15737                 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
15738                 /*
15739                  * Check if we should limit by maximum payload
15740                  * length:
15741                  */
15742                 if (if_hw_tsomax != 0) {
15743                         /* compute maximum TSO length */
15744                         max_len = (if_hw_tsomax - hdrlen -
15745                                    max_linkhdr);
15746                         if (max_len <= 0) {
15747                                 goto failed;
15748                         } else if (len > max_len) {
15749                                 len = max_len;
15750                         }
15751                 }
15752                 if (len <= segsiz) {
15753                         /*
15754                          * In case there are too many small fragments don't
15755                          * use TSO:
15756                          */
15757                         tso = 0;
15758                 }
15759         } else {
15760                 tso = 0;
15761         }
15762         if ((tso == 0) && (len > segsiz))
15763                 len = segsiz;
15764         if ((len == 0) ||
15765             (len <= MHLEN - hdrlen - max_linkhdr)) {
15766                 goto failed;
15767         }
15768         th->th_seq = htonl(rsm->r_start);
15769         th->th_ack = htonl(tp->rcv_nxt);
15770         /*
15771          * The PUSH bit should only be applied
15772          * if the full retransmission is made. If
15773          * we are sending less than this is the
15774          * left hand edge and should not have
15775          * the PUSH bit.
15776          */
15777         if ((rsm->r_flags & RACK_HAD_PUSH) &&
15778             (len == (rsm->r_end - rsm->r_start)))
15779                 flags |= TH_PUSH;
15780         th->th_win = htons((u_short)(rack->r_ctl.fsb.recwin >> tp->rcv_scale));
15781         if (th->th_win == 0) {
15782                 tp->t_sndzerowin++;
15783                 tp->t_flags |= TF_RXWIN0SENT;
15784         } else
15785                 tp->t_flags &= ~TF_RXWIN0SENT;
15786         if (rsm->r_flags & RACK_TLP) {
15787                 /*
15788                  * TLP should not count in retran count, but
15789                  * in its own bin
15790                  */
15791                 counter_u64_add(rack_tlp_retran, 1);
15792                 counter_u64_add(rack_tlp_retran_bytes, len);
15793         } else {
15794                 tp->t_sndrexmitpack++;
15795                 KMOD_TCPSTAT_INC(tcps_sndrexmitpack);
15796                 KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len);
15797         }
15798 #ifdef STATS
15799         stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
15800                                  len);
15801 #endif
15802         if (rsm->m == NULL)
15803                 goto failed;
15804         if (rsm->orig_m_len != rsm->m->m_len) {
15805                 /* Fix up the orig_m_len and possibly the mbuf offset */
15806                 rack_adjust_orig_mlen(rsm);
15807         }
15808         m->m_next = rack_fo_base_copym(rsm->m, rsm->soff, &len, NULL, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, rsm->r_hw_tls);
15809         if (len <= segsiz) {
15810                 /*
15811                  * Must have ran out of mbufs for the copy
15812                  * shorten it to no longer need tso. Lets
15813                  * not put on sendalot since we are low on
15814                  * mbufs.
15815                  */
15816                 tso = 0;
15817         }
15818         if ((m->m_next == NULL) || (len <= 0)){
15819                 goto failed;
15820         }
15821         if (udp) {
15822                 if (rack->r_is_v6)
15823                         ulen = hdrlen + len - sizeof(struct ip6_hdr);
15824                 else
15825                         ulen = hdrlen + len - sizeof(struct ip);
15826                 udp->uh_ulen = htons(ulen);
15827         }
15828         m->m_pkthdr.rcvif = (struct ifnet *)0;
15829         if (TCPS_HAVERCVDSYN(tp->t_state) &&
15830             (tp->t_flags2 & TF2_ECN_PERMIT)) {
15831                 int ect = tcp_ecn_output_established(tp, &flags, len, true);
15832                 if ((tp->t_state == TCPS_SYN_RECEIVED) &&
15833                     (tp->t_flags2 & TF2_ECN_SND_ECE))
15834                     tp->t_flags2 &= ~TF2_ECN_SND_ECE;
15835 #ifdef INET6
15836                 if (rack->r_is_v6) {
15837                     ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << 20);
15838                     ip6->ip6_flow |= htonl(ect << 20);
15839                 }
15840                 else
15841 #endif
15842                 {
15843                     ip->ip_tos &= ~IPTOS_ECN_MASK;
15844                     ip->ip_tos |= ect;
15845                 }
15846         }
15847         tcp_set_flags(th, flags);
15848         m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
15849 #ifdef INET6
15850         if (rack->r_is_v6) {
15851                 if (tp->t_port) {
15852                         m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
15853                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
15854                         udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
15855                         th->th_sum = htons(0);
15856                         UDPSTAT_INC(udps_opackets);
15857                 } else {
15858                         m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
15859                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
15860                         th->th_sum = in6_cksum_pseudo(ip6,
15861                                                       sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP,
15862                                                       0);
15863                 }
15864         }
15865 #endif
15866 #if defined(INET6) && defined(INET)
15867         else
15868 #endif
15869 #ifdef INET
15870         {
15871                 if (tp->t_port) {
15872                         m->m_pkthdr.csum_flags = CSUM_UDP;
15873                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
15874                         udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
15875                                                 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
15876                         th->th_sum = htons(0);
15877                         UDPSTAT_INC(udps_opackets);
15878                 } else {
15879                         m->m_pkthdr.csum_flags = CSUM_TCP;
15880                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
15881                         th->th_sum = in_pseudo(ip->ip_src.s_addr,
15882                                                ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
15883                                                                         IPPROTO_TCP + len + optlen));
15884                 }
15885                 /* IP version must be set here for ipv4/ipv6 checking later */
15886                 KASSERT(ip->ip_v == IPVERSION,
15887                         ("%s: IP version incorrect: %d", __func__, ip->ip_v));
15888         }
15889 #endif
15890         if (tso) {
15891                 KASSERT(len > tp->t_maxseg - optlen,
15892                         ("%s: len <= tso_segsz tp:%p", __func__, tp));
15893                 m->m_pkthdr.csum_flags |= CSUM_TSO;
15894                 m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen;
15895         }
15896 #ifdef INET6
15897         if (rack->r_is_v6) {
15898                 ip6->ip6_hlim = rack->r_ctl.fsb.hoplimit;
15899                 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
15900                 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss)
15901                         tp->t_flags2 |= TF2_PLPMTU_PMTUD;
15902                 else
15903                         tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
15904         }
15905 #endif
15906 #if defined(INET) && defined(INET6)
15907         else
15908 #endif
15909 #ifdef INET
15910         {
15911                 ip->ip_len = htons(m->m_pkthdr.len);
15912                 ip->ip_ttl = rack->r_ctl.fsb.hoplimit;
15913                 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
15914                         tp->t_flags2 |= TF2_PLPMTU_PMTUD;
15915                         if (tp->t_port == 0 || len < V_tcp_minmss) {
15916                                 ip->ip_off |= htons(IP_DF);
15917                         }
15918                 } else {
15919                         tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
15920                 }
15921         }
15922 #endif
15923         /* Time to copy in our header */
15924         cpto = mtod(m, uint8_t *);
15925         memcpy(cpto, rack->r_ctl.fsb.tcp_ip_hdr, rack->r_ctl.fsb.tcp_ip_hdr_len);
15926         th = (struct tcphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.th - rack->r_ctl.fsb.tcp_ip_hdr));
15927         if (optlen) {
15928                 bcopy(opt, th + 1, optlen);
15929                 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
15930         } else {
15931                 th->th_off = sizeof(struct tcphdr) >> 2;
15932         }
15933         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
15934                 union tcp_log_stackspecific log;
15935
15936                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
15937                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
15938                 if (rack->rack_no_prr)
15939                         log.u_bbr.flex1 = 0;
15940                 else
15941                         log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
15942                 log.u_bbr.flex2 = rack->r_ctl.rc_pace_min_segs;
15943                 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs;
15944                 log.u_bbr.flex4 = max_val;
15945                 log.u_bbr.flex5 = 0;
15946                 /* Save off the early/late values */
15947                 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early;
15948                 log.u_bbr.applimited = rack->r_ctl.rc_agg_delayed;
15949                 log.u_bbr.bw_inuse = rack_get_bw(rack);
15950                 if (doing_tlp == 0)
15951                         log.u_bbr.flex8 = 1;
15952                 else
15953                         log.u_bbr.flex8 = 2;
15954                 log.u_bbr.pacing_gain = rack_get_output_gain(rack, NULL);
15955                 log.u_bbr.flex7 = 55;
15956                 log.u_bbr.pkts_out = tp->t_maxseg;
15957                 log.u_bbr.timeStamp = cts;
15958                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
15959                 log.u_bbr.lt_epoch = rack->r_ctl.cwnd_to_use;
15960                 log.u_bbr.delivered = 0;
15961                 lgb = tcp_log_event_(tp, th, NULL, NULL, TCP_LOG_OUT, ERRNO_UNK,
15962                                      len, &log, false, NULL, NULL, 0, tv);
15963         } else
15964                 lgb = NULL;
15965 #ifdef INET6
15966         if (rack->r_is_v6) {
15967                 error = ip6_output(m, NULL,
15968                                    &inp->inp_route6,
15969                                    0, NULL, NULL, inp);
15970         }
15971 #endif
15972 #if defined(INET) && defined(INET6)
15973         else
15974 #endif
15975 #ifdef INET
15976         {
15977                 error = ip_output(m, NULL,
15978                                   &inp->inp_route,
15979                                   0, 0, inp);
15980         }
15981 #endif
15982         m = NULL;
15983         if (lgb) {
15984                 lgb->tlb_errno = error;
15985                 lgb = NULL;
15986         }
15987         if (error) {
15988                 goto failed;
15989         }
15990         rack_log_output(tp, &to, len, rsm->r_start, flags, error, rack_to_usec_ts(tv),
15991                         rsm, RACK_SENT_FP, rsm->m, rsm->soff, rsm->r_hw_tls);
15992         if (doing_tlp && (rack->fast_rsm_hack == 0)) {
15993                 rack->rc_tlp_in_progress = 1;
15994                 rack->r_ctl.rc_tlp_cnt_out++;
15995         }
15996         if (error == 0) {
15997                 tcp_account_for_send(tp, len, 1, doing_tlp, rsm->r_hw_tls);
15998                 if (doing_tlp) {
15999                         rack->rc_last_sent_tlp_past_cumack = 0;
16000                         rack->rc_last_sent_tlp_seq_valid = 1;
16001                         rack->r_ctl.last_sent_tlp_seq = rsm->r_start;
16002                         rack->r_ctl.last_sent_tlp_len = rsm->r_end - rsm->r_start;
16003                 }
16004         }
16005         tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
16006         rack->forced_ack = 0;   /* If we send something zap the FA flag */
16007         if (IN_FASTRECOVERY(tp->t_flags) && rsm)
16008                 rack->r_ctl.retran_during_recovery += len;
16009         {
16010                 int idx;
16011
16012                 idx = (len / segsiz) + 3;
16013                 if (idx >= TCP_MSS_ACCT_ATIMER)
16014                         counter_u64_add(rack_out_size[(TCP_MSS_ACCT_ATIMER-1)], 1);
16015                 else
16016                         counter_u64_add(rack_out_size[idx], 1);
16017         }
16018         if (tp->t_rtttime == 0) {
16019                 tp->t_rtttime = ticks;
16020                 tp->t_rtseq = startseq;
16021                 KMOD_TCPSTAT_INC(tcps_segstimed);
16022         }
16023         counter_u64_add(rack_fto_rsm_send, 1);
16024         if (error && (error == ENOBUFS)) {
16025                 if (rack->r_ctl.crte != NULL) {
16026                         rack_trace_point(rack, RACK_TP_HWENOBUF);
16027                 } else
16028                         rack_trace_point(rack, RACK_TP_ENOBUF);
16029                 slot = ((1 + rack->rc_enobuf) * HPTS_USEC_IN_MSEC);
16030                 if (rack->rc_enobuf < 0x7f)
16031                         rack->rc_enobuf++;
16032                 if (slot < (10 * HPTS_USEC_IN_MSEC))
16033                         slot = 10 * HPTS_USEC_IN_MSEC;
16034         } else
16035                 slot = rack_get_pacing_delay(rack, tp, len, NULL, segsiz);
16036         if ((slot == 0) ||
16037             (rack->rc_always_pace == 0) ||
16038             (rack->r_rr_config == 1)) {
16039                 /*
16040                  * We have no pacing set or we
16041                  * are using old-style rack or
16042                  * we are overriden to use the old 1ms pacing.
16043                  */
16044                 slot = rack->r_ctl.rc_min_to;
16045         }
16046         rack_start_hpts_timer(rack, tp, cts, slot, len, 0);
16047 #ifdef TCP_ACCOUNTING
16048         crtsc = get_cyclecount();
16049         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16050                 tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru;
16051         }
16052         counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], cnt_thru);
16053         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16054                 tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val);
16055         }
16056         counter_u64_add(tcp_proc_time[SND_OUT_DATA], (crtsc - ts_val));
16057         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16058                 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((len + segsiz - 1) / segsiz);
16059         }
16060         counter_u64_add(tcp_cnt_counters[CNT_OF_MSS_OUT], ((len + segsiz - 1) / segsiz));
16061         sched_unpin();
16062 #endif
16063         return (0);
16064 failed:
16065         if (m)
16066                 m_free(m);
16067         return (-1);
16068 }
16069
16070 static void
16071 rack_sndbuf_autoscale(struct tcp_rack *rack)
16072 {
16073         /*
16074          * Automatic sizing of send socket buffer.  Often the send buffer
16075          * size is not optimally adjusted to the actual network conditions
16076          * at hand (delay bandwidth product).  Setting the buffer size too
16077          * small limits throughput on links with high bandwidth and high
16078          * delay (eg. trans-continental/oceanic links).  Setting the
16079          * buffer size too big consumes too much real kernel memory,
16080          * especially with many connections on busy servers.
16081          *
16082          * The criteria to step up the send buffer one notch are:
16083          *  1. receive window of remote host is larger than send buffer
16084          *     (with a fudge factor of 5/4th);
16085          *  2. send buffer is filled to 7/8th with data (so we actually
16086          *     have data to make use of it);
16087          *  3. send buffer fill has not hit maximal automatic size;
16088          *  4. our send window (slow start and cogestion controlled) is
16089          *     larger than sent but unacknowledged data in send buffer.
16090          *
16091          * Note that the rack version moves things much faster since
16092          * we want to avoid hitting cache lines in the rack_fast_output()
16093          * path so this is called much less often and thus moves
16094          * the SB forward by a percentage.
16095          */
16096         struct socket *so;
16097         struct tcpcb *tp;
16098         uint32_t sendwin, scaleup;
16099
16100         tp = rack->rc_tp;
16101         so = rack->rc_inp->inp_socket;
16102         sendwin = min(rack->r_ctl.cwnd_to_use, tp->snd_wnd);
16103         if (V_tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE) {
16104                 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat &&
16105                     sbused(&so->so_snd) >=
16106                     (so->so_snd.sb_hiwat / 8 * 7) &&
16107                     sbused(&so->so_snd) < V_tcp_autosndbuf_max &&
16108                     sendwin >= (sbused(&so->so_snd) -
16109                     (tp->snd_nxt - tp->snd_una))) {
16110                         if (rack_autosndbuf_inc)
16111                                 scaleup = (rack_autosndbuf_inc * so->so_snd.sb_hiwat) / 100;
16112                         else
16113                                 scaleup = V_tcp_autosndbuf_inc;
16114                         if (scaleup < V_tcp_autosndbuf_inc)
16115                                 scaleup = V_tcp_autosndbuf_inc;
16116                         scaleup += so->so_snd.sb_hiwat;
16117                         if (scaleup > V_tcp_autosndbuf_max)
16118                                 scaleup = V_tcp_autosndbuf_max;
16119                         if (!sbreserve_locked(so, SO_SND, scaleup, curthread))
16120                                 so->so_snd.sb_flags &= ~SB_AUTOSIZE;
16121                 }
16122         }
16123 }
16124
16125 static int
16126 rack_fast_output(struct tcpcb *tp, struct tcp_rack *rack, uint64_t ts_val,
16127                  uint32_t cts, uint32_t ms_cts, struct timeval *tv, long tot_len, int *send_err)
16128 {
16129         /*
16130          * Enter to do fast output. We are given that the sched_pin is
16131          * in place (if accounting is compiled in) and the cycle count taken
16132          * at entry is in place in ts_val. The idea here is that
16133          * we know how many more bytes needs to be sent (presumably either
16134          * during pacing or to fill the cwnd and that was greater than
16135          * the max-burst). We have how much to send and all the info we
16136          * need to just send.
16137          */
16138         struct ip *ip = NULL;
16139         struct udphdr *udp = NULL;
16140         struct tcphdr *th = NULL;
16141         struct mbuf *m, *s_mb;
16142         struct inpcb *inp;
16143         uint8_t *cpto;
16144         struct tcp_log_buffer *lgb;
16145 #ifdef TCP_ACCOUNTING
16146         uint64_t crtsc;
16147 #endif
16148         struct tcpopt to;
16149         u_char opt[TCP_MAXOLEN];
16150         uint32_t hdrlen, optlen;
16151         int cnt_thru = 1;
16152         int32_t slot, segsiz, len, max_val, tso = 0, sb_offset, error, ulen = 0;
16153         uint16_t flags;
16154         uint32_t s_soff;
16155         uint32_t if_hw_tsomaxsegcount = 0, startseq;
16156         uint32_t if_hw_tsomaxsegsize;
16157         uint16_t add_flag = RACK_SENT_FP;
16158 #ifdef INET6
16159         struct ip6_hdr *ip6 = NULL;
16160
16161         if (rack->r_is_v6) {
16162                 ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr;
16163                 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
16164         } else
16165 #endif                          /* INET6 */
16166         {
16167                 ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
16168                 hdrlen = sizeof(struct tcpiphdr);
16169         }
16170         if (tp->t_port && (V_tcp_udp_tunneling_port == 0)) {
16171                 m = NULL;
16172                 goto failed;
16173         }
16174         startseq = tp->snd_max;
16175         segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
16176         inp = rack->rc_inp;
16177         len = rack->r_ctl.fsb.left_to_send;
16178         to.to_flags = 0;
16179         flags = rack->r_ctl.fsb.tcp_flags;
16180         if (tp->t_flags & TF_RCVD_TSTMP) {
16181                 to.to_tsval = ms_cts + tp->ts_offset;
16182                 to.to_tsecr = tp->ts_recent;
16183                 to.to_flags = TOF_TS;
16184         }
16185         optlen = tcp_addoptions(&to, opt);
16186         hdrlen += optlen;
16187         udp = rack->r_ctl.fsb.udp;
16188         if (udp)
16189                 hdrlen += sizeof(struct udphdr);
16190         if (rack->r_ctl.rc_pace_max_segs)
16191                 max_val = rack->r_ctl.rc_pace_max_segs;
16192         else if (rack->rc_user_set_max_segs)
16193                 max_val = rack->rc_user_set_max_segs * segsiz;
16194         else
16195                 max_val = len;
16196         if ((tp->t_flags & TF_TSO) &&
16197             V_tcp_do_tso &&
16198             (len > segsiz) &&
16199             (tp->t_port == 0))
16200                 tso = 1;
16201 again:
16202 #ifdef INET6
16203         if (MHLEN < hdrlen + max_linkhdr)
16204                 m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
16205         else
16206 #endif
16207                 m = m_gethdr(M_NOWAIT, MT_DATA);
16208         if (m == NULL)
16209                 goto failed;
16210         m->m_data += max_linkhdr;
16211         m->m_len = hdrlen;
16212         th = rack->r_ctl.fsb.th;
16213         /* Establish the len to send */
16214         if (len > max_val)
16215                 len = max_val;
16216         if ((tso) && (len + optlen > tp->t_maxseg)) {
16217                 uint32_t if_hw_tsomax;
16218                 int32_t max_len;
16219
16220                 /* extract TSO information */
16221                 if_hw_tsomax = tp->t_tsomax;
16222                 if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
16223                 if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
16224                 /*
16225                  * Check if we should limit by maximum payload
16226                  * length:
16227                  */
16228                 if (if_hw_tsomax != 0) {
16229                         /* compute maximum TSO length */
16230                         max_len = (if_hw_tsomax - hdrlen -
16231                                    max_linkhdr);
16232                         if (max_len <= 0) {
16233                                 goto failed;
16234                         } else if (len > max_len) {
16235                                 len = max_len;
16236                         }
16237                 }
16238                 if (len <= segsiz) {
16239                         /*
16240                          * In case there are too many small fragments don't
16241                          * use TSO:
16242                          */
16243                         tso = 0;
16244                 }
16245         } else {
16246                 tso = 0;
16247         }
16248         if ((tso == 0) && (len > segsiz))
16249                 len = segsiz;
16250         if ((len == 0) ||
16251             (len <= MHLEN - hdrlen - max_linkhdr)) {
16252                 goto failed;
16253         }
16254         sb_offset = tp->snd_max - tp->snd_una;
16255         th->th_seq = htonl(tp->snd_max);
16256         th->th_ack = htonl(tp->rcv_nxt);
16257         th->th_win = htons((u_short)(rack->r_ctl.fsb.recwin >> tp->rcv_scale));
16258         if (th->th_win == 0) {
16259                 tp->t_sndzerowin++;
16260                 tp->t_flags |= TF_RXWIN0SENT;
16261         } else
16262                 tp->t_flags &= ~TF_RXWIN0SENT;
16263         tp->snd_up = tp->snd_una;       /* drag it along, its deprecated */
16264         KMOD_TCPSTAT_INC(tcps_sndpack);
16265         KMOD_TCPSTAT_ADD(tcps_sndbyte, len);
16266 #ifdef STATS
16267         stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
16268                                  len);
16269 #endif
16270         if (rack->r_ctl.fsb.m == NULL)
16271                 goto failed;
16272
16273         /* s_mb and s_soff are saved for rack_log_output */
16274         m->m_next = rack_fo_m_copym(rack, &len, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize,
16275                                     &s_mb, &s_soff);
16276         if (len <= segsiz) {
16277                 /*
16278                  * Must have ran out of mbufs for the copy
16279                  * shorten it to no longer need tso. Lets
16280                  * not put on sendalot since we are low on
16281                  * mbufs.
16282                  */
16283                 tso = 0;
16284         }
16285         if (rack->r_ctl.fsb.rfo_apply_push &&
16286             (len == rack->r_ctl.fsb.left_to_send)) {
16287                 flags |= TH_PUSH;
16288                 add_flag |= RACK_HAD_PUSH;
16289         }
16290         if ((m->m_next == NULL) || (len <= 0)){
16291                 goto failed;
16292         }
16293         if (udp) {
16294                 if (rack->r_is_v6)
16295                         ulen = hdrlen + len - sizeof(struct ip6_hdr);
16296                 else
16297                         ulen = hdrlen + len - sizeof(struct ip);
16298                 udp->uh_ulen = htons(ulen);
16299         }
16300         m->m_pkthdr.rcvif = (struct ifnet *)0;
16301         if (TCPS_HAVERCVDSYN(tp->t_state) &&
16302             (tp->t_flags2 & TF2_ECN_PERMIT)) {
16303                 int ect = tcp_ecn_output_established(tp, &flags, len, false);
16304                 if ((tp->t_state == TCPS_SYN_RECEIVED) &&
16305                     (tp->t_flags2 & TF2_ECN_SND_ECE))
16306                         tp->t_flags2 &= ~TF2_ECN_SND_ECE;
16307 #ifdef INET6
16308                 if (rack->r_is_v6) {
16309                         ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << 20);
16310                         ip6->ip6_flow |= htonl(ect << 20);
16311                 }
16312                 else
16313 #endif
16314                 {
16315                         ip->ip_tos &= ~IPTOS_ECN_MASK;
16316                         ip->ip_tos |= ect;
16317                 }
16318         }
16319         tcp_set_flags(th, flags);
16320         m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
16321 #ifdef INET6
16322         if (rack->r_is_v6) {
16323                 if (tp->t_port) {
16324                         m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
16325                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
16326                         udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
16327                         th->th_sum = htons(0);
16328                         UDPSTAT_INC(udps_opackets);
16329                 } else {
16330                         m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
16331                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
16332                         th->th_sum = in6_cksum_pseudo(ip6,
16333                                                       sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP,
16334                                                       0);
16335                 }
16336         }
16337 #endif
16338 #if defined(INET6) && defined(INET)
16339         else
16340 #endif
16341 #ifdef INET
16342         {
16343                 if (tp->t_port) {
16344                         m->m_pkthdr.csum_flags = CSUM_UDP;
16345                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
16346                         udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
16347                                                 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
16348                         th->th_sum = htons(0);
16349                         UDPSTAT_INC(udps_opackets);
16350                 } else {
16351                         m->m_pkthdr.csum_flags = CSUM_TCP;
16352                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
16353                         th->th_sum = in_pseudo(ip->ip_src.s_addr,
16354                                                ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
16355                                                                         IPPROTO_TCP + len + optlen));
16356                 }
16357                 /* IP version must be set here for ipv4/ipv6 checking later */
16358                 KASSERT(ip->ip_v == IPVERSION,
16359                         ("%s: IP version incorrect: %d", __func__, ip->ip_v));
16360         }
16361 #endif
16362         if (tso) {
16363                 KASSERT(len > tp->t_maxseg - optlen,
16364                         ("%s: len <= tso_segsz tp:%p", __func__, tp));
16365                 m->m_pkthdr.csum_flags |= CSUM_TSO;
16366                 m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen;
16367         }
16368 #ifdef INET6
16369         if (rack->r_is_v6) {
16370                 ip6->ip6_hlim = rack->r_ctl.fsb.hoplimit;
16371                 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
16372                 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss)
16373                         tp->t_flags2 |= TF2_PLPMTU_PMTUD;
16374                 else
16375                         tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
16376         }
16377 #endif
16378 #if defined(INET) && defined(INET6)
16379         else
16380 #endif
16381 #ifdef INET
16382         {
16383                 ip->ip_len = htons(m->m_pkthdr.len);
16384                 ip->ip_ttl = rack->r_ctl.fsb.hoplimit;
16385                 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
16386                         tp->t_flags2 |= TF2_PLPMTU_PMTUD;
16387                         if (tp->t_port == 0 || len < V_tcp_minmss) {
16388                                 ip->ip_off |= htons(IP_DF);
16389                         }
16390                 } else {
16391                         tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
16392                 }
16393         }
16394 #endif
16395         /* Time to copy in our header */
16396         cpto = mtod(m, uint8_t *);
16397         memcpy(cpto, rack->r_ctl.fsb.tcp_ip_hdr, rack->r_ctl.fsb.tcp_ip_hdr_len);
16398         th = (struct tcphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.th - rack->r_ctl.fsb.tcp_ip_hdr));
16399         if (optlen) {
16400                 bcopy(opt, th + 1, optlen);
16401                 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
16402         } else {
16403                 th->th_off = sizeof(struct tcphdr) >> 2;
16404         }
16405         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
16406                 union tcp_log_stackspecific log;
16407
16408                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
16409                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
16410                 if (rack->rack_no_prr)
16411                         log.u_bbr.flex1 = 0;
16412                 else
16413                         log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
16414                 log.u_bbr.flex2 = rack->r_ctl.rc_pace_min_segs;
16415                 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs;
16416                 log.u_bbr.flex4 = max_val;
16417                 log.u_bbr.flex5 = 0;
16418                 /* Save off the early/late values */
16419                 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early;
16420                 log.u_bbr.applimited = rack->r_ctl.rc_agg_delayed;
16421                 log.u_bbr.bw_inuse = rack_get_bw(rack);
16422                 log.u_bbr.flex8 = 0;
16423                 log.u_bbr.pacing_gain = rack_get_output_gain(rack, NULL);
16424                 log.u_bbr.flex7 = 44;
16425                 log.u_bbr.pkts_out = tp->t_maxseg;
16426                 log.u_bbr.timeStamp = cts;
16427                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
16428                 log.u_bbr.lt_epoch = rack->r_ctl.cwnd_to_use;
16429                 log.u_bbr.delivered = 0;
16430                 lgb = tcp_log_event_(tp, th, NULL, NULL, TCP_LOG_OUT, ERRNO_UNK,
16431                                      len, &log, false, NULL, NULL, 0, tv);
16432         } else
16433                 lgb = NULL;
16434 #ifdef INET6
16435         if (rack->r_is_v6) {
16436                 error = ip6_output(m, NULL,
16437                                    &inp->inp_route6,
16438                                    0, NULL, NULL, inp);
16439         }
16440 #endif
16441 #if defined(INET) && defined(INET6)
16442         else
16443 #endif
16444 #ifdef INET
16445         {
16446                 error = ip_output(m, NULL,
16447                                   &inp->inp_route,
16448                                   0, 0, inp);
16449         }
16450 #endif
16451         if (lgb) {
16452                 lgb->tlb_errno = error;
16453                 lgb = NULL;
16454         }
16455         if (error) {
16456                 *send_err = error;
16457                 m = NULL;
16458                 goto failed;
16459         }
16460         rack_log_output(tp, &to, len, tp->snd_max, flags, error, rack_to_usec_ts(tv),
16461                         NULL, add_flag, s_mb, s_soff, rack->r_ctl.fsb.hw_tls);
16462         m = NULL;
16463         if (tp->snd_una == tp->snd_max) {
16464                 rack->r_ctl.rc_tlp_rxt_last_time = cts;
16465                 rack_log_progress_event(rack, tp, ticks, PROGRESS_START, __LINE__);
16466                 tp->t_acktime = ticks;
16467         }
16468         if (error == 0)
16469                 tcp_account_for_send(tp, len, 0, 0, rack->r_ctl.fsb.hw_tls);
16470
16471         rack->forced_ack = 0;   /* If we send something zap the FA flag */
16472         tot_len += len;
16473         if ((tp->t_flags & TF_GPUTINPROG) == 0)
16474                 rack_start_gp_measurement(tp, rack, tp->snd_max, sb_offset);
16475         tp->snd_max += len;
16476         tp->snd_nxt = tp->snd_max;
16477         {
16478                 int idx;
16479
16480                 idx = (len / segsiz) + 3;
16481                 if (idx >= TCP_MSS_ACCT_ATIMER)
16482                         counter_u64_add(rack_out_size[(TCP_MSS_ACCT_ATIMER-1)], 1);
16483                 else
16484                         counter_u64_add(rack_out_size[idx], 1);
16485         }
16486         if (len <= rack->r_ctl.fsb.left_to_send)
16487                 rack->r_ctl.fsb.left_to_send -= len;
16488         else
16489                 rack->r_ctl.fsb.left_to_send = 0;
16490         if (rack->r_ctl.fsb.left_to_send < segsiz) {
16491                 rack->r_fast_output = 0;
16492                 rack->r_ctl.fsb.left_to_send = 0;
16493                 /* At the end of fast_output scale up the sb */
16494                 SOCKBUF_LOCK(&rack->rc_inp->inp_socket->so_snd);
16495                 rack_sndbuf_autoscale(rack);
16496                 SOCKBUF_UNLOCK(&rack->rc_inp->inp_socket->so_snd);
16497         }
16498         if (tp->t_rtttime == 0) {
16499                 tp->t_rtttime = ticks;
16500                 tp->t_rtseq = startseq;
16501                 KMOD_TCPSTAT_INC(tcps_segstimed);
16502         }
16503         if ((rack->r_ctl.fsb.left_to_send >= segsiz) &&
16504             (max_val > len) &&
16505             (tso == 0)) {
16506                 max_val -= len;
16507                 len = segsiz;
16508                 th = rack->r_ctl.fsb.th;
16509                 cnt_thru++;
16510                 goto again;
16511         }
16512         tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
16513         counter_u64_add(rack_fto_send, 1);
16514         slot = rack_get_pacing_delay(rack, tp, tot_len, NULL, segsiz);
16515         rack_start_hpts_timer(rack, tp, cts, slot, tot_len, 0);
16516 #ifdef TCP_ACCOUNTING
16517         crtsc = get_cyclecount();
16518         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16519                 tp->tcp_cnt_counters[SND_OUT_DATA] += cnt_thru;
16520         }
16521         counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], cnt_thru);
16522         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16523                 tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val);
16524         }
16525         counter_u64_add(tcp_proc_time[SND_OUT_DATA], (crtsc - ts_val));
16526         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16527                 tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((tot_len + segsiz - 1) / segsiz);
16528         }
16529         counter_u64_add(tcp_cnt_counters[CNT_OF_MSS_OUT], ((tot_len + segsiz - 1) / segsiz));
16530         sched_unpin();
16531 #endif
16532         return (0);
16533 failed:
16534         if (m)
16535                 m_free(m);
16536         rack->r_fast_output = 0;
16537         return (-1);
16538 }
16539
16540 static int
16541 rack_output(struct tcpcb *tp)
16542 {
16543         struct socket *so;
16544         uint32_t recwin;
16545         uint32_t sb_offset, s_moff = 0;
16546         int32_t len, error = 0;
16547         uint16_t flags;
16548         struct mbuf *m, *s_mb = NULL;
16549         struct mbuf *mb;
16550         uint32_t if_hw_tsomaxsegcount = 0;
16551         uint32_t if_hw_tsomaxsegsize;
16552         int32_t segsiz, minseg;
16553         long tot_len_this_send = 0;
16554 #ifdef INET
16555         struct ip *ip = NULL;
16556 #endif
16557         struct udphdr *udp = NULL;
16558         struct tcp_rack *rack;
16559         struct tcphdr *th;
16560         uint8_t pass = 0;
16561         uint8_t mark = 0;
16562         uint8_t wanted_cookie = 0;
16563         u_char opt[TCP_MAXOLEN];
16564         unsigned ipoptlen, optlen, hdrlen, ulen=0;
16565         uint32_t rack_seq;
16566
16567 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
16568         unsigned ipsec_optlen = 0;
16569
16570 #endif
16571         int32_t idle, sendalot;
16572         int32_t sub_from_prr = 0;
16573         volatile int32_t sack_rxmit;
16574         struct rack_sendmap *rsm = NULL;
16575         int32_t tso, mtu;
16576         struct tcpopt to;
16577         int32_t slot = 0;
16578         int32_t sup_rack = 0;
16579         uint32_t cts, ms_cts, delayed, early;
16580         uint16_t add_flag = RACK_SENT_SP;
16581         /* The doing_tlp flag will be set by the actual rack_timeout_tlp() */
16582         uint8_t hpts_calling,  doing_tlp = 0;
16583         uint32_t cwnd_to_use, pace_max_seg;
16584         int32_t do_a_prefetch = 0;
16585         int32_t prefetch_rsm = 0;
16586         int32_t orig_len = 0;
16587         struct timeval tv;
16588         int32_t prefetch_so_done = 0;
16589         struct tcp_log_buffer *lgb;
16590         struct inpcb *inp;
16591         struct sockbuf *sb;
16592         uint64_t ts_val = 0;
16593 #ifdef TCP_ACCOUNTING
16594         uint64_t crtsc;
16595 #endif
16596 #ifdef INET6
16597         struct ip6_hdr *ip6 = NULL;
16598         int32_t isipv6;
16599 #endif
16600         uint8_t filled_all = 0;
16601         bool hw_tls = false;
16602
16603         /* setup and take the cache hits here */
16604         rack = (struct tcp_rack *)tp->t_fb_ptr;
16605 #ifdef TCP_ACCOUNTING
16606         sched_pin();
16607         ts_val = get_cyclecount();
16608 #endif
16609         hpts_calling = rack->rc_inp->inp_hpts_calls;
16610         NET_EPOCH_ASSERT();
16611         INP_WLOCK_ASSERT(rack->rc_inp);
16612 #ifdef TCP_OFFLOAD
16613         if (tp->t_flags & TF_TOE) {
16614 #ifdef TCP_ACCOUNTING
16615                 sched_unpin();
16616 #endif
16617                 return (tcp_offload_output(tp));
16618         }
16619 #endif
16620         /*
16621          * For TFO connections in SYN_RECEIVED, only allow the initial
16622          * SYN|ACK and those sent by the retransmit timer.
16623          */
16624         if (IS_FASTOPEN(tp->t_flags) &&
16625             (tp->t_state == TCPS_SYN_RECEIVED) &&
16626             SEQ_GT(tp->snd_max, tp->snd_una) &&    /* initial SYN|ACK sent */
16627             (rack->r_ctl.rc_resend == NULL)) {         /* not a retransmit */
16628 #ifdef TCP_ACCOUNTING
16629                 sched_unpin();
16630 #endif
16631                 return (0);
16632         }
16633 #ifdef INET6
16634         if (rack->r_state) {
16635                 /* Use the cache line loaded if possible */
16636                 isipv6 = rack->r_is_v6;
16637         } else {
16638                 isipv6 = (rack->rc_inp->inp_vflag & INP_IPV6) != 0;
16639         }
16640 #endif
16641         early = 0;
16642         cts = tcp_get_usecs(&tv);
16643         ms_cts = tcp_tv_to_mssectick(&tv);
16644         if (((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
16645             tcp_in_hpts(rack->rc_inp)) {
16646                 /*
16647                  * We are on the hpts for some timer but not hptsi output.
16648                  * Remove from the hpts unconditionally.
16649                  */
16650                 rack_timer_cancel(tp, rack, cts, __LINE__);
16651         }
16652         /* Are we pacing and late? */
16653         if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
16654             TSTMP_GEQ(cts, rack->r_ctl.rc_last_output_to)) {
16655                 /* We are delayed */
16656                 delayed = cts - rack->r_ctl.rc_last_output_to;
16657         } else {
16658                 delayed = 0;
16659         }
16660         /* Do the timers, which may override the pacer */
16661         if (rack->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
16662                 int retval;
16663
16664                 retval = rack_process_timers(tp, rack, cts, hpts_calling,
16665                     &doing_tlp);
16666                 if (retval != 0) {
16667                         counter_u64_add(rack_out_size[TCP_MSS_ACCT_ATIMER], 1);
16668 #ifdef TCP_ACCOUNTING
16669                         sched_unpin();
16670 #endif
16671                         /*
16672                          * If timers want tcp_drop(), then pass error out,
16673                          * otherwise suppress it.
16674                          */
16675                         return (retval < 0 ? retval : 0);
16676                 }
16677         }
16678         if (rack->rc_in_persist) {
16679                 if (tcp_in_hpts(rack->rc_inp) == 0) {
16680                         /* Timer is not running */
16681                         rack_start_hpts_timer(rack, tp, cts, 0, 0, 0);
16682                 }
16683 #ifdef TCP_ACCOUNTING
16684                 sched_unpin();
16685 #endif
16686                 return (0);
16687         }
16688         if ((rack->r_timer_override) ||
16689             (rack->rc_ack_can_sendout_data) ||
16690             (delayed) ||
16691             (tp->t_state < TCPS_ESTABLISHED)) {
16692                 rack->rc_ack_can_sendout_data = 0;
16693                 if (tcp_in_hpts(rack->rc_inp))
16694                         tcp_hpts_remove(rack->rc_inp);
16695         } else if (tcp_in_hpts(rack->rc_inp)) {
16696                 /*
16697                  * On the hpts you can't pass even if ACKNOW is on, we will
16698                  * when the hpts fires.
16699                  */
16700 #ifdef TCP_ACCOUNTING
16701                 crtsc = get_cyclecount();
16702                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16703                         tp->tcp_proc_time[SND_BLOCKED] += (crtsc - ts_val);
16704                 }
16705                 counter_u64_add(tcp_proc_time[SND_BLOCKED], (crtsc - ts_val));
16706                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
16707                         tp->tcp_cnt_counters[SND_BLOCKED]++;
16708                 }
16709                 counter_u64_add(tcp_cnt_counters[SND_BLOCKED], 1);
16710                 sched_unpin();
16711 #endif
16712                 counter_u64_add(rack_out_size[TCP_MSS_ACCT_INPACE], 1);
16713                 return (0);
16714         }
16715         rack->rc_inp->inp_hpts_calls = 0;
16716         /* Finish out both pacing early and late accounting */
16717         if ((rack->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
16718             TSTMP_GT(rack->r_ctl.rc_last_output_to, cts)) {
16719                 early = rack->r_ctl.rc_last_output_to - cts;
16720         } else
16721                 early = 0;
16722         if (delayed) {
16723                 rack->r_ctl.rc_agg_delayed += delayed;
16724                 rack->r_late = 1;
16725         } else if (early) {
16726                 rack->r_ctl.rc_agg_early += early;
16727                 rack->r_early = 1;
16728         }
16729         /* Now that early/late accounting is done turn off the flag */
16730         rack->r_ctl.rc_hpts_flags &= ~PACE_PKT_OUTPUT;
16731         rack->r_wanted_output = 0;
16732         rack->r_timer_override = 0;
16733         if ((tp->t_state != rack->r_state) &&
16734             TCPS_HAVEESTABLISHED(tp->t_state)) {
16735                 rack_set_state(tp, rack);
16736         }
16737         if ((rack->r_fast_output) &&
16738             (doing_tlp == 0) &&
16739             (tp->rcv_numsacks == 0)) {
16740                 int ret;
16741
16742                 error = 0;
16743                 ret = rack_fast_output(tp, rack, ts_val, cts, ms_cts, &tv, tot_len_this_send, &error);
16744                 if (ret >= 0)
16745                         return(ret);
16746                 else if (error) {
16747                         inp = rack->rc_inp;
16748                         so = inp->inp_socket;
16749                         sb = &so->so_snd;
16750                         goto nomore;
16751                 }
16752         }
16753         inp = rack->rc_inp;
16754         /*
16755          * For TFO connections in SYN_SENT or SYN_RECEIVED,
16756          * only allow the initial SYN or SYN|ACK and those sent
16757          * by the retransmit timer.
16758          */
16759         if (IS_FASTOPEN(tp->t_flags) &&
16760             ((tp->t_state == TCPS_SYN_RECEIVED) ||
16761              (tp->t_state == TCPS_SYN_SENT)) &&
16762             SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */
16763             (tp->t_rxtshift == 0)) {              /* not a retransmit */
16764                 cwnd_to_use = rack->r_ctl.cwnd_to_use = tp->snd_cwnd;
16765                 so = inp->inp_socket;
16766                 sb = &so->so_snd;
16767                 goto just_return_nolock;
16768         }
16769         /*
16770          * Determine length of data that should be transmitted, and flags
16771          * that will be used. If there is some data or critical controls
16772          * (SYN, RST) to send, then transmit; otherwise, investigate
16773          * further.
16774          */
16775         idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
16776         if (tp->t_idle_reduce) {
16777                 if (idle && (TICKS_2_USEC(ticks - tp->t_rcvtime) >= tp->t_rxtcur))
16778                         rack_cc_after_idle(rack, tp);
16779         }
16780         tp->t_flags &= ~TF_LASTIDLE;
16781         if (idle) {
16782                 if (tp->t_flags & TF_MORETOCOME) {
16783                         tp->t_flags |= TF_LASTIDLE;
16784                         idle = 0;
16785                 }
16786         }
16787         if ((tp->snd_una == tp->snd_max) &&
16788             rack->r_ctl.rc_went_idle_time &&
16789             TSTMP_GT(cts, rack->r_ctl.rc_went_idle_time)) {
16790                 idle = cts - rack->r_ctl.rc_went_idle_time;
16791                 if (idle > rack_min_probertt_hold) {
16792                         /* Count as a probe rtt */
16793                         if (rack->in_probe_rtt == 0) {
16794                                 rack->r_ctl.rc_lower_rtt_us_cts = cts;
16795                                 rack->r_ctl.rc_time_probertt_entered = rack->r_ctl.rc_lower_rtt_us_cts;
16796                                 rack->r_ctl.rc_time_probertt_starts = rack->r_ctl.rc_lower_rtt_us_cts;
16797                                 rack->r_ctl.rc_time_of_last_probertt = rack->r_ctl.rc_lower_rtt_us_cts;
16798                         } else {
16799                                 rack_exit_probertt(rack, cts);
16800                         }
16801                 }
16802                 idle = 0;
16803         }
16804         if (rack_use_fsb && (rack->r_fsb_inited == 0) && (rack->r_state != TCPS_CLOSED))
16805                 rack_init_fsb_block(tp, rack);
16806 again:
16807         /*
16808          * If we've recently taken a timeout, snd_max will be greater than
16809          * snd_nxt.  There may be SACK information that allows us to avoid
16810          * resending already delivered data.  Adjust snd_nxt accordingly.
16811          */
16812         sendalot = 0;
16813         cts = tcp_get_usecs(&tv);
16814         ms_cts = tcp_tv_to_mssectick(&tv);
16815         tso = 0;
16816         mtu = 0;
16817         segsiz = min(ctf_fixed_maxseg(tp), rack->r_ctl.rc_pace_min_segs);
16818         minseg = segsiz;
16819         if (rack->r_ctl.rc_pace_max_segs == 0)
16820                 pace_max_seg = rack->rc_user_set_max_segs * segsiz;
16821         else
16822                 pace_max_seg = rack->r_ctl.rc_pace_max_segs;
16823         sb_offset = tp->snd_max - tp->snd_una;
16824         cwnd_to_use = rack->r_ctl.cwnd_to_use = tp->snd_cwnd;
16825         flags = tcp_outflags[tp->t_state];
16826         while (rack->rc_free_cnt < rack_free_cache) {
16827                 rsm = rack_alloc(rack);
16828                 if (rsm == NULL) {
16829                         if (inp->inp_hpts_calls)
16830                                 /* Retry in a ms */
16831                                 slot = (1 * HPTS_USEC_IN_MSEC);
16832                         so = inp->inp_socket;
16833                         sb = &so->so_snd;
16834                         goto just_return_nolock;
16835                 }
16836                 TAILQ_INSERT_TAIL(&rack->r_ctl.rc_free, rsm, r_tnext);
16837                 rack->rc_free_cnt++;
16838                 rsm = NULL;
16839         }
16840         if (inp->inp_hpts_calls)
16841                 inp->inp_hpts_calls = 0;
16842         sack_rxmit = 0;
16843         len = 0;
16844         rsm = NULL;
16845         if (flags & TH_RST) {
16846                 SOCKBUF_LOCK(&inp->inp_socket->so_snd);
16847                 so = inp->inp_socket;
16848                 sb = &so->so_snd;
16849                 goto send;
16850         }
16851         if (rack->r_ctl.rc_resend) {
16852                 /* Retransmit timer */
16853                 rsm = rack->r_ctl.rc_resend;
16854                 rack->r_ctl.rc_resend = NULL;
16855                 len = rsm->r_end - rsm->r_start;
16856                 sack_rxmit = 1;
16857                 sendalot = 0;
16858                 KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start),
16859                         ("%s:%d: r.start:%u < SND.UNA:%u; tp:%p, rack:%p, rsm:%p",
16860                          __func__, __LINE__,
16861                          rsm->r_start, tp->snd_una, tp, rack, rsm));
16862                 sb_offset = rsm->r_start - tp->snd_una;
16863                 if (len >= segsiz)
16864                         len = segsiz;
16865         } else if ((rsm = tcp_rack_output(tp, rack, cts)) != NULL) {
16866                 /* We have a retransmit that takes precedence */
16867                 if ((!IN_FASTRECOVERY(tp->t_flags)) &&
16868                     ((rsm->r_flags & RACK_MUST_RXT) == 0) &&
16869                     ((tp->t_flags & TF_WASFRECOVERY) == 0)) {
16870                         /* Enter recovery if not induced by a time-out */
16871                         rack_cong_signal(tp, CC_NDUPACK, tp->snd_una, __LINE__);
16872                 }
16873 #ifdef INVARIANTS
16874                 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
16875                         panic("Huh, tp:%p rack:%p rsm:%p start:%u < snd_una:%u\n",
16876                               tp, rack, rsm, rsm->r_start, tp->snd_una);
16877                 }
16878 #endif
16879                 len = rsm->r_end - rsm->r_start;
16880                 KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start),
16881                         ("%s:%d: r.start:%u < SND.UNA:%u; tp:%p, rack:%p, rsm:%p",
16882                          __func__, __LINE__,
16883                          rsm->r_start, tp->snd_una, tp, rack, rsm));
16884                 sb_offset = rsm->r_start - tp->snd_una;
16885                 sendalot = 0;
16886                 if (len >= segsiz)
16887                         len = segsiz;
16888                 if (len > 0) {
16889                         sack_rxmit = 1;
16890                         KMOD_TCPSTAT_INC(tcps_sack_rexmits);
16891                         KMOD_TCPSTAT_ADD(tcps_sack_rexmit_bytes,
16892                             min(len, segsiz));
16893                 }
16894         } else if (rack->r_ctl.rc_tlpsend) {
16895                 /* Tail loss probe */
16896                 long cwin;
16897                 long tlen;
16898
16899                 /*
16900                  * Check if we can do a TLP with a RACK'd packet
16901                  * this can happen if we are not doing the rack
16902                  * cheat and we skipped to a TLP and it
16903                  * went off.
16904                  */
16905                 rsm = rack->r_ctl.rc_tlpsend;
16906                 /* We are doing a TLP make sure the flag is preent */
16907                 rsm->r_flags |= RACK_TLP;
16908                 rack->r_ctl.rc_tlpsend = NULL;
16909                 sack_rxmit = 1;
16910                 tlen = rsm->r_end - rsm->r_start;
16911                 if (tlen > segsiz)
16912                         tlen = segsiz;
16913                 KASSERT(SEQ_LEQ(tp->snd_una, rsm->r_start),
16914                         ("%s:%d: r.start:%u < SND.UNA:%u; tp:%p, rack:%p, rsm:%p",
16915                          __func__, __LINE__,
16916                          rsm->r_start, tp->snd_una, tp, rack, rsm));
16917                 sb_offset = rsm->r_start - tp->snd_una;
16918                 cwin = min(tp->snd_wnd, tlen);
16919                 len = cwin;
16920         }
16921         if (rack->r_must_retran &&
16922             (doing_tlp == 0) &&
16923             (rsm == NULL)) {
16924                 /*
16925                  * Non-Sack and we had a RTO or Sack/non-Sack and a
16926                  * MTU change, we need to retransmit until we reach
16927                  * the former snd_max (rack->r_ctl.rc_snd_max_at_rto).
16928                  */
16929                 if (SEQ_GT(tp->snd_max, tp->snd_una)) {
16930                         int sendwin, flight;
16931
16932                         sendwin = min(tp->snd_wnd, tp->snd_cwnd);
16933                         flight = ctf_flight_size(tp, rack->r_ctl.rc_out_at_rto);
16934                         if (flight >= sendwin) {
16935                                 so = inp->inp_socket;
16936                                 sb = &so->so_snd;
16937                                 goto just_return_nolock;
16938                         }
16939                         rsm = TAILQ_FIRST(&rack->r_ctl.rc_tmap);
16940                         if (rsm == NULL) {
16941                                 /* TSNH */
16942                                 rack->r_must_retran = 0;
16943                                 rack->r_ctl.rc_out_at_rto = 0;
16944                                 so = inp->inp_socket;
16945                                 sb = &so->so_snd;
16946                                 goto just_return_nolock;
16947                         }
16948                         if ((rsm->r_flags & RACK_MUST_RXT) == 0) {
16949                                 /* It does not have the flag, we are done */
16950                                 rack->r_must_retran = 0;
16951                                 rack->r_ctl.rc_out_at_rto = 0;
16952                         } else {
16953                                 sack_rxmit = 1;
16954                                 len = rsm->r_end - rsm->r_start;
16955                                 sendalot = 0;
16956                                 sb_offset = rsm->r_start - tp->snd_una;
16957                                 if (len >= segsiz)
16958                                         len = segsiz;
16959                                 /*
16960                                  * Delay removing the flag RACK_MUST_RXT so
16961                                  * that the fastpath for retransmit will
16962                                  * work with this rsm.
16963                                  */
16964
16965                         }
16966                 } else {
16967                         /* We must be done if there is nothing outstanding */
16968                         rack->r_must_retran = 0;
16969                         rack->r_ctl.rc_out_at_rto = 0;
16970                 }
16971         }
16972         /*
16973          * Enforce a connection sendmap count limit if set
16974          * as long as we are not retransmiting.
16975          */
16976         if ((rsm == NULL) &&
16977             (rack->do_detection == 0) &&
16978             (V_tcp_map_entries_limit > 0) &&
16979             (rack->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
16980                 counter_u64_add(rack_to_alloc_limited, 1);
16981                 if (!rack->alloc_limit_reported) {
16982                         rack->alloc_limit_reported = 1;
16983                         counter_u64_add(rack_alloc_limited_conns, 1);
16984                 }
16985                 so = inp->inp_socket;
16986                 sb = &so->so_snd;
16987                 goto just_return_nolock;
16988         }
16989         if (rsm && (rsm->r_flags & RACK_HAS_FIN)) {
16990                 /* we are retransmitting the fin */
16991                 len--;
16992                 if (len) {
16993                         /*
16994                          * When retransmitting data do *not* include the
16995                          * FIN. This could happen from a TLP probe.
16996                          */
16997                         flags &= ~TH_FIN;
16998                 }
16999         }
17000         if (rsm && rack->r_fsb_inited && rack_use_rsm_rfo &&
17001             ((rsm->r_flags & RACK_HAS_FIN) == 0)) {
17002                 int ret;
17003
17004                 ret = rack_fast_rsm_output(tp, rack, rsm, ts_val, cts, ms_cts, &tv, len, doing_tlp);
17005                 if (ret == 0)
17006                         return (0);
17007         }
17008         so = inp->inp_socket;
17009         sb = &so->so_snd;
17010         if (do_a_prefetch == 0) {
17011                 kern_prefetch(sb, &do_a_prefetch);
17012                 do_a_prefetch = 1;
17013         }
17014 #ifdef NETFLIX_SHARED_CWND
17015         if ((tp->t_flags2 & TF2_TCP_SCWND_ALLOWED) &&
17016             rack->rack_enable_scwnd) {
17017                 /* We are doing cwnd sharing */
17018                 if (rack->gp_ready &&
17019                     (rack->rack_attempted_scwnd == 0) &&
17020                     (rack->r_ctl.rc_scw == NULL) &&
17021                     tp->t_lib) {
17022                         /* The pcbid is in, lets make an attempt */
17023                         counter_u64_add(rack_try_scwnd, 1);
17024                         rack->rack_attempted_scwnd = 1;
17025                         rack->r_ctl.rc_scw = tcp_shared_cwnd_alloc(tp,
17026                                                                    &rack->r_ctl.rc_scw_index,
17027                                                                    segsiz);
17028                 }
17029                 if (rack->r_ctl.rc_scw &&
17030                     (rack->rack_scwnd_is_idle == 1) &&
17031                     sbavail(&so->so_snd)) {
17032                         /* we are no longer out of data */
17033                         tcp_shared_cwnd_active(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index);
17034                         rack->rack_scwnd_is_idle = 0;
17035                 }
17036                 if (rack->r_ctl.rc_scw) {
17037                         /* First lets update and get the cwnd */
17038                         rack->r_ctl.cwnd_to_use = cwnd_to_use = tcp_shared_cwnd_update(rack->r_ctl.rc_scw,
17039                                                                     rack->r_ctl.rc_scw_index,
17040                                                                     tp->snd_cwnd, tp->snd_wnd, segsiz);
17041                 }
17042         }
17043 #endif
17044         /*
17045          * Get standard flags, and add SYN or FIN if requested by 'hidden'
17046          * state flags.
17047          */
17048         if (tp->t_flags & TF_NEEDFIN)
17049                 flags |= TH_FIN;
17050         if (tp->t_flags & TF_NEEDSYN)
17051                 flags |= TH_SYN;
17052         if ((sack_rxmit == 0) && (prefetch_rsm == 0)) {
17053                 void *end_rsm;
17054                 end_rsm = TAILQ_LAST_FAST(&rack->r_ctl.rc_tmap, rack_sendmap, r_tnext);
17055                 if (end_rsm)
17056                         kern_prefetch(end_rsm, &prefetch_rsm);
17057                 prefetch_rsm = 1;
17058         }
17059         SOCKBUF_LOCK(sb);
17060         /*
17061          * If snd_nxt == snd_max and we have transmitted a FIN, the
17062          * sb_offset will be > 0 even if so_snd.sb_cc is 0, resulting in a
17063          * negative length.  This can also occur when TCP opens up its
17064          * congestion window while receiving additional duplicate acks after
17065          * fast-retransmit because TCP will reset snd_nxt to snd_max after
17066          * the fast-retransmit.
17067          *
17068          * In the normal retransmit-FIN-only case, however, snd_nxt will be
17069          * set to snd_una, the sb_offset will be 0, and the length may wind
17070          * up 0.
17071          *
17072          * If sack_rxmit is true we are retransmitting from the scoreboard
17073          * in which case len is already set.
17074          */
17075         if ((sack_rxmit == 0) &&
17076             (TCPS_HAVEESTABLISHED(tp->t_state) || IS_FASTOPEN(tp->t_flags))) {
17077                 uint32_t avail;
17078
17079                 avail = sbavail(sb);
17080                 if (SEQ_GT(tp->snd_nxt, tp->snd_una) && avail)
17081                         sb_offset = tp->snd_nxt - tp->snd_una;
17082                 else
17083                         sb_offset = 0;
17084                 if ((IN_FASTRECOVERY(tp->t_flags) == 0) || rack->rack_no_prr) {
17085                         if (rack->r_ctl.rc_tlp_new_data) {
17086                                 /* TLP is forcing out new data */
17087                                 if (rack->r_ctl.rc_tlp_new_data > (uint32_t) (avail - sb_offset)) {
17088                                         rack->r_ctl.rc_tlp_new_data = (uint32_t) (avail - sb_offset);
17089                                 }
17090                                 if ((rack->r_ctl.rc_tlp_new_data + sb_offset) > tp->snd_wnd) {
17091                                         if (tp->snd_wnd > sb_offset)
17092                                                 len = tp->snd_wnd - sb_offset;
17093                                         else
17094                                                 len = 0;
17095                                 } else {
17096                                         len = rack->r_ctl.rc_tlp_new_data;
17097                                 }
17098                                 rack->r_ctl.rc_tlp_new_data = 0;
17099                         }  else {
17100                                 len = rack_what_can_we_send(tp, rack, cwnd_to_use, avail, sb_offset);
17101                         }
17102                         if ((rack->r_ctl.crte == NULL) && IN_FASTRECOVERY(tp->t_flags) && (len > segsiz)) {
17103                                 /*
17104                                  * For prr=off, we need to send only 1 MSS
17105                                  * at a time. We do this because another sack could
17106                                  * be arriving that causes us to send retransmits and
17107                                  * we don't want to be on a long pace due to a larger send
17108                                  * that keeps us from sending out the retransmit.
17109                                  */
17110                                 len = segsiz;
17111                         }
17112                 } else {
17113                         uint32_t outstanding;
17114                         /*
17115                          * We are inside of a Fast recovery episode, this
17116                          * is caused by a SACK or 3 dup acks. At this point
17117                          * we have sent all the retransmissions and we rely
17118                          * on PRR to dictate what we will send in the form of
17119                          * new data.
17120                          */
17121
17122                         outstanding = tp->snd_max - tp->snd_una;
17123                         if ((rack->r_ctl.rc_prr_sndcnt + outstanding) > tp->snd_wnd) {
17124                                 if (tp->snd_wnd > outstanding) {
17125                                         len = tp->snd_wnd - outstanding;
17126                                         /* Check to see if we have the data */
17127                                         if ((sb_offset + len) > avail) {
17128                                                 /* It does not all fit */
17129                                                 if (avail > sb_offset)
17130                                                         len = avail - sb_offset;
17131                                                 else
17132                                                         len = 0;
17133                                         }
17134                                 } else {
17135                                         len = 0;
17136                                 }
17137                         } else if (avail > sb_offset) {
17138                                 len = avail - sb_offset;
17139                         } else {
17140                                 len = 0;
17141                         }
17142                         if (len > 0) {
17143                                 if (len > rack->r_ctl.rc_prr_sndcnt) {
17144                                         len = rack->r_ctl.rc_prr_sndcnt;
17145                                 }
17146                                 if (len > 0) {
17147                                         sub_from_prr = 1;
17148                                 }
17149                         }
17150                         if (len > segsiz) {
17151                                 /*
17152                                  * We should never send more than a MSS when
17153                                  * retransmitting or sending new data in prr
17154                                  * mode unless the override flag is on. Most
17155                                  * likely the PRR algorithm is not going to
17156                                  * let us send a lot as well :-)
17157                                  */
17158                                 if (rack->r_ctl.rc_prr_sendalot == 0) {
17159                                         len = segsiz;
17160                                 }
17161                         } else if (len < segsiz) {
17162                                 /*
17163                                  * Do we send any? The idea here is if the
17164                                  * send empty's the socket buffer we want to
17165                                  * do it. However if not then lets just wait
17166                                  * for our prr_sndcnt to get bigger.
17167                                  */
17168                                 long leftinsb;
17169
17170                                 leftinsb = sbavail(sb) - sb_offset;
17171                                 if (leftinsb > len) {
17172                                         /* This send does not empty the sb */
17173                                         len = 0;
17174                                 }
17175                         }
17176                 }
17177         } else if (!TCPS_HAVEESTABLISHED(tp->t_state)) {
17178                 /*
17179                  * If you have not established
17180                  * and are not doing FAST OPEN
17181                  * no data please.
17182                  */
17183                 if ((sack_rxmit == 0) &&
17184                     (!IS_FASTOPEN(tp->t_flags))){
17185                         len = 0;
17186                         sb_offset = 0;
17187                 }
17188         }
17189         if (prefetch_so_done == 0) {
17190                 kern_prefetch(so, &prefetch_so_done);
17191                 prefetch_so_done = 1;
17192         }
17193         /*
17194          * Lop off SYN bit if it has already been sent.  However, if this is
17195          * SYN-SENT state and if segment contains data and if we don't know
17196          * that foreign host supports TAO, suppress sending segment.
17197          */
17198         if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una) &&
17199             ((sack_rxmit == 0) && (tp->t_rxtshift == 0))) {
17200                 /*
17201                  * When sending additional segments following a TFO SYN|ACK,
17202                  * do not include the SYN bit.
17203                  */
17204                 if (IS_FASTOPEN(tp->t_flags) &&
17205                     (tp->t_state == TCPS_SYN_RECEIVED))
17206                         flags &= ~TH_SYN;
17207         }
17208         /*
17209          * Be careful not to send data and/or FIN on SYN segments. This
17210          * measure is needed to prevent interoperability problems with not
17211          * fully conformant TCP implementations.
17212          */
17213         if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
17214                 len = 0;
17215                 flags &= ~TH_FIN;
17216         }
17217         /*
17218          * On TFO sockets, ensure no data is sent in the following cases:
17219          *
17220          *  - When retransmitting SYN|ACK on a passively-created socket
17221          *
17222          *  - When retransmitting SYN on an actively created socket
17223          *
17224          *  - When sending a zero-length cookie (cookie request) on an
17225          *    actively created socket
17226          *
17227          *  - When the socket is in the CLOSED state (RST is being sent)
17228          */
17229         if (IS_FASTOPEN(tp->t_flags) &&
17230             (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||
17231              ((tp->t_state == TCPS_SYN_SENT) &&
17232               (tp->t_tfo_client_cookie_len == 0)) ||
17233              (flags & TH_RST))) {
17234                 sack_rxmit = 0;
17235                 len = 0;
17236         }
17237         /* Without fast-open there should never be data sent on a SYN */
17238         if ((flags & TH_SYN) && (!IS_FASTOPEN(tp->t_flags))) {
17239                 tp->snd_nxt = tp->iss;
17240                 len = 0;
17241         }
17242         if ((len > segsiz) && (tcp_dsack_block_exists(tp))) {
17243                 /* We only send 1 MSS if we have a DSACK block */
17244                 add_flag |= RACK_SENT_W_DSACK;
17245                 len = segsiz;
17246         }
17247         orig_len = len;
17248         if (len <= 0) {
17249                 /*
17250                  * If FIN has been sent but not acked, but we haven't been
17251                  * called to retransmit, len will be < 0.  Otherwise, window
17252                  * shrank after we sent into it.  If window shrank to 0,
17253                  * cancel pending retransmit, pull snd_nxt back to (closed)
17254                  * window, and set the persist timer if it isn't already
17255                  * going.  If the window didn't close completely, just wait
17256                  * for an ACK.
17257                  *
17258                  * We also do a general check here to ensure that we will
17259                  * set the persist timer when we have data to send, but a
17260                  * 0-byte window. This makes sure the persist timer is set
17261                  * even if the packet hits one of the "goto send" lines
17262                  * below.
17263                  */
17264                 len = 0;
17265                 if ((tp->snd_wnd == 0) &&
17266                     (TCPS_HAVEESTABLISHED(tp->t_state)) &&
17267                     (tp->snd_una == tp->snd_max) &&
17268                     (sb_offset < (int)sbavail(sb))) {
17269                         rack_enter_persist(tp, rack, cts);
17270                 }
17271         } else if ((rsm == NULL) &&
17272                    (doing_tlp == 0) &&
17273                    (len < pace_max_seg)) {
17274                 /*
17275                  * We are not sending a maximum sized segment for
17276                  * some reason. Should we not send anything (think
17277                  * sws or persists)?
17278                  */
17279                 if ((tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), minseg)) &&
17280                     (TCPS_HAVEESTABLISHED(tp->t_state)) &&
17281                     (len < minseg) &&
17282                     (len < (int)(sbavail(sb) - sb_offset))) {
17283                         /*
17284                          * Here the rwnd is less than
17285                          * the minimum pacing size, this is not a retransmit,
17286                          * we are established and
17287                          * the send is not the last in the socket buffer
17288                          * we send nothing, and we may enter persists
17289                          * if nothing is outstanding.
17290                          */
17291                         len = 0;
17292                         if (tp->snd_max == tp->snd_una) {
17293                                 /*
17294                                  * Nothing out we can
17295                                  * go into persists.
17296                                  */
17297                                 rack_enter_persist(tp, rack, cts);
17298                         }
17299                      } else if ((cwnd_to_use >= max(minseg, (segsiz * 4))) &&
17300                            (ctf_flight_size(tp, rack->r_ctl.rc_sacked) > (2 * segsiz)) &&
17301                            (len < (int)(sbavail(sb) - sb_offset)) &&
17302                            (len < minseg)) {
17303                         /*
17304                          * Here we are not retransmitting, and
17305                          * the cwnd is not so small that we could
17306                          * not send at least a min size (rxt timer
17307                          * not having gone off), We have 2 segments or
17308                          * more already in flight, its not the tail end
17309                          * of the socket buffer  and the cwnd is blocking
17310                          * us from sending out a minimum pacing segment size.
17311                          * Lets not send anything.
17312                          */
17313                         len = 0;
17314                 } else if (((tp->snd_wnd - ctf_outstanding(tp)) <
17315                             min((rack->r_ctl.rc_high_rwnd/2), minseg)) &&
17316                            (ctf_flight_size(tp, rack->r_ctl.rc_sacked) > (2 * segsiz)) &&
17317                            (len < (int)(sbavail(sb) - sb_offset)) &&
17318                            (TCPS_HAVEESTABLISHED(tp->t_state))) {
17319                         /*
17320                          * Here we have a send window but we have
17321                          * filled it up and we can't send another pacing segment.
17322                          * We also have in flight more than 2 segments
17323                          * and we are not completing the sb i.e. we allow
17324                          * the last bytes of the sb to go out even if
17325                          * its not a full pacing segment.
17326                          */
17327                         len = 0;
17328                 } else if ((rack->r_ctl.crte != NULL) &&
17329                            (tp->snd_wnd >= (pace_max_seg * max(1, rack_hw_rwnd_factor))) &&
17330                            (cwnd_to_use >= (pace_max_seg + (4 * segsiz))) &&
17331                            (ctf_flight_size(tp, rack->r_ctl.rc_sacked) >= (2 * segsiz)) &&
17332                            (len < (int)(sbavail(sb) - sb_offset))) {
17333                         /*
17334                          * Here we are doing hardware pacing, this is not a TLP,
17335                          * we are not sending a pace max segment size, there is rwnd
17336                          * room to send at least N pace_max_seg, the cwnd is greater
17337                          * than or equal to a full pacing segments plus 4 mss and we have 2 or
17338                          * more segments in flight and its not the tail of the socket buffer.
17339                          *
17340                          * We don't want to send instead we need to get more ack's in to
17341                          * allow us to send a full pacing segment. Normally, if we are pacing
17342                          * about the right speed, we should have finished our pacing
17343                          * send as most of the acks have come back if we are at the
17344                          * right rate. This is a bit fuzzy since return path delay
17345                          * can delay the acks, which is why we want to make sure we
17346                          * have cwnd space to have a bit more than a max pace segments in flight.
17347                          *
17348                          * If we have not gotten our acks back we are pacing at too high a
17349                          * rate delaying will not hurt and will bring our GP estimate down by
17350                          * injecting the delay. If we don't do this we will send
17351                          * 2 MSS out in response to the acks being clocked in which
17352                          * defeats the point of hw-pacing (i.e. to help us get
17353                          * larger TSO's out).
17354                          */
17355                         len = 0;
17356
17357                 }
17358
17359         }
17360         /* len will be >= 0 after this point. */
17361         KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
17362         rack_sndbuf_autoscale(rack);
17363         /*
17364          * Decide if we can use TCP Segmentation Offloading (if supported by
17365          * hardware).
17366          *
17367          * TSO may only be used if we are in a pure bulk sending state.  The
17368          * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP
17369          * options prevent using TSO.  With TSO the TCP header is the same
17370          * (except for the sequence number) for all generated packets.  This
17371          * makes it impossible to transmit any options which vary per
17372          * generated segment or packet.
17373          *
17374          * IPv4 handling has a clear separation of ip options and ip header
17375          * flags while IPv6 combines both in in6p_outputopts. ip6_optlen() does
17376          * the right thing below to provide length of just ip options and thus
17377          * checking for ipoptlen is enough to decide if ip options are present.
17378          */
17379         ipoptlen = 0;
17380 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
17381         /*
17382          * Pre-calculate here as we save another lookup into the darknesses
17383          * of IPsec that way and can actually decide if TSO is ok.
17384          */
17385 #ifdef INET6
17386         if (isipv6 && IPSEC_ENABLED(ipv6))
17387                 ipsec_optlen = IPSEC_HDRSIZE(ipv6, tp->t_inpcb);
17388 #ifdef INET
17389         else
17390 #endif
17391 #endif                          /* INET6 */
17392 #ifdef INET
17393                 if (IPSEC_ENABLED(ipv4))
17394                         ipsec_optlen = IPSEC_HDRSIZE(ipv4, tp->t_inpcb);
17395 #endif                          /* INET */
17396 #endif
17397
17398 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
17399         ipoptlen += ipsec_optlen;
17400 #endif
17401         if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && len > segsiz &&
17402             (tp->t_port == 0) &&
17403             ((tp->t_flags & TF_SIGNATURE) == 0) &&
17404             tp->rcv_numsacks == 0 && sack_rxmit == 0 &&
17405             ipoptlen == 0)
17406                 tso = 1;
17407         {
17408                 uint32_t outstanding;
17409
17410                 outstanding = tp->snd_max - tp->snd_una;
17411                 if (tp->t_flags & TF_SENTFIN) {
17412                         /*
17413                          * If we sent a fin, snd_max is 1 higher than
17414                          * snd_una
17415                          */
17416                         outstanding--;
17417                 }
17418                 if (sack_rxmit) {
17419                         if ((rsm->r_flags & RACK_HAS_FIN) == 0)
17420                                 flags &= ~TH_FIN;
17421                 } else {
17422                         if (SEQ_LT(tp->snd_nxt + len, tp->snd_una +
17423                                    sbused(sb)))
17424                                 flags &= ~TH_FIN;
17425                 }
17426         }
17427         recwin = lmin(lmax(sbspace(&so->so_rcv), 0),
17428             (long)TCP_MAXWIN << tp->rcv_scale);
17429
17430         /*
17431          * Sender silly window avoidance.   We transmit under the following
17432          * conditions when len is non-zero:
17433          *
17434          * - We have a full segment (or more with TSO) - This is the last
17435          * buffer in a write()/send() and we are either idle or running
17436          * NODELAY - we've timed out (e.g. persist timer) - we have more
17437          * then 1/2 the maximum send window's worth of data (receiver may be
17438          * limited the window size) - we need to retransmit
17439          */
17440         if (len) {
17441                 if (len >= segsiz) {
17442                         goto send;
17443                 }
17444                 /*
17445                  * NOTE! on localhost connections an 'ack' from the remote
17446                  * end may occur synchronously with the output and cause us
17447                  * to flush a buffer queued with moretocome.  XXX
17448                  *
17449                  */
17450                 if (!(tp->t_flags & TF_MORETOCOME) &&   /* normal case */
17451                     (idle || (tp->t_flags & TF_NODELAY)) &&
17452                     ((uint32_t)len + (uint32_t)sb_offset >= sbavail(sb)) &&
17453                     (tp->t_flags & TF_NOPUSH) == 0) {
17454                         pass = 2;
17455                         goto send;
17456                 }
17457                 if ((tp->snd_una == tp->snd_max) && len) {      /* Nothing outstanding */
17458                         pass = 22;
17459                         goto send;
17460                 }
17461                 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
17462                         pass = 4;
17463                         goto send;
17464                 }
17465                 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) { /* retransmit case */
17466                         pass = 5;
17467                         goto send;
17468                 }
17469                 if (sack_rxmit) {
17470                         pass = 6;
17471                         goto send;
17472                 }
17473                 if (((tp->snd_wnd - ctf_outstanding(tp)) < segsiz) &&
17474                     (ctf_outstanding(tp) < (segsiz * 2))) {
17475                         /*
17476                          * We have less than two MSS outstanding (delayed ack)
17477                          * and our rwnd will not let us send a full sized
17478                          * MSS. Lets go ahead and let this small segment
17479                          * out because we want to try to have at least two
17480                          * packets inflight to not be caught by delayed ack.
17481                          */
17482                         pass = 12;
17483                         goto send;
17484                 }
17485         }
17486         /*
17487          * Sending of standalone window updates.
17488          *
17489          * Window updates are important when we close our window due to a
17490          * full socket buffer and are opening it again after the application
17491          * reads data from it.  Once the window has opened again and the
17492          * remote end starts to send again the ACK clock takes over and
17493          * provides the most current window information.
17494          *
17495          * We must avoid the silly window syndrome whereas every read from
17496          * the receive buffer, no matter how small, causes a window update
17497          * to be sent.  We also should avoid sending a flurry of window
17498          * updates when the socket buffer had queued a lot of data and the
17499          * application is doing small reads.
17500          *
17501          * Prevent a flurry of pointless window updates by only sending an
17502          * update when we can increase the advertized window by more than
17503          * 1/4th of the socket buffer capacity.  When the buffer is getting
17504          * full or is very small be more aggressive and send an update
17505          * whenever we can increase by two mss sized segments. In all other
17506          * situations the ACK's to new incoming data will carry further
17507          * window increases.
17508          *
17509          * Don't send an independent window update if a delayed ACK is
17510          * pending (it will get piggy-backed on it) or the remote side
17511          * already has done a half-close and won't send more data.  Skip
17512          * this if the connection is in T/TCP half-open state.
17513          */
17514         if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&
17515             !(tp->t_flags & TF_DELACK) &&
17516             !TCPS_HAVERCVDFIN(tp->t_state)) {
17517                 /*
17518                  * "adv" is the amount we could increase the window, taking
17519                  * into account that we are limited by TCP_MAXWIN <<
17520                  * tp->rcv_scale.
17521                  */
17522                 int32_t adv;
17523                 int oldwin;
17524
17525                 adv = recwin;
17526                 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
17527                         oldwin = (tp->rcv_adv - tp->rcv_nxt);
17528                         if (adv > oldwin)
17529                             adv -= oldwin;
17530                         else {
17531                                 /* We can't increase the window */
17532                                 adv = 0;
17533                         }
17534                 } else
17535                         oldwin = 0;
17536
17537                 /*
17538                  * If the new window size ends up being the same as or less
17539                  * than the old size when it is scaled, then don't force
17540                  * a window update.
17541                  */
17542                 if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale)
17543                         goto dontupdate;
17544
17545                 if (adv >= (int32_t)(2 * segsiz) &&
17546                     (adv >= (int32_t)(so->so_rcv.sb_hiwat / 4) ||
17547                      recwin <= (int32_t)(so->so_rcv.sb_hiwat / 8) ||
17548                      so->so_rcv.sb_hiwat <= 8 * segsiz)) {
17549                         pass = 7;
17550                         goto send;
17551                 }
17552                 if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat) {
17553                         pass = 23;
17554                         goto send;
17555                 }
17556         }
17557 dontupdate:
17558
17559         /*
17560          * Send if we owe the peer an ACK, RST, SYN, or urgent data.  ACKNOW
17561          * is also a catch-all for the retransmit timer timeout case.
17562          */
17563         if (tp->t_flags & TF_ACKNOW) {
17564                 pass = 8;
17565                 goto send;
17566         }
17567         if (((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0)) {
17568                 pass = 9;
17569                 goto send;
17570         }
17571         /*
17572          * If our state indicates that FIN should be sent and we have not
17573          * yet done so, then we need to send.
17574          */
17575         if ((flags & TH_FIN) &&
17576             (tp->snd_nxt == tp->snd_una)) {
17577                 pass = 11;
17578                 goto send;
17579         }
17580         /*
17581          * No reason to send a segment, just return.
17582          */
17583 just_return:
17584         SOCKBUF_UNLOCK(sb);
17585 just_return_nolock:
17586         {
17587                 int app_limited = CTF_JR_SENT_DATA;
17588
17589                 if (tot_len_this_send > 0) {
17590                         /* Make sure snd_nxt is up to max */
17591                         rack->r_ctl.fsb.recwin = recwin;
17592                         slot = rack_get_pacing_delay(rack, tp, tot_len_this_send, NULL, segsiz);
17593                         if ((error == 0) &&
17594                             rack_use_rfo &&
17595                             ((flags & (TH_SYN|TH_FIN)) == 0) &&
17596                             (ipoptlen == 0) &&
17597                             (tp->snd_nxt == tp->snd_max) &&
17598                             (tp->rcv_numsacks == 0) &&
17599                             rack->r_fsb_inited &&
17600                             TCPS_HAVEESTABLISHED(tp->t_state) &&
17601                             (rack->r_must_retran == 0) &&
17602                             ((tp->t_flags & TF_NEEDFIN) == 0) &&
17603                             (len > 0) && (orig_len > 0) &&
17604                             (orig_len > len) &&
17605                             ((orig_len - len) >= segsiz) &&
17606                             ((optlen == 0) ||
17607                              ((optlen == TCPOLEN_TSTAMP_APPA) && (to.to_flags & TOF_TS)))) {
17608                                 /* We can send at least one more MSS using our fsb */
17609
17610                                 rack->r_fast_output = 1;
17611                                 rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off);
17612                                 rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len;
17613                                 rack->r_ctl.fsb.tcp_flags = flags;
17614                                 rack->r_ctl.fsb.left_to_send = orig_len - len;
17615                                 if (hw_tls)
17616                                         rack->r_ctl.fsb.hw_tls = 1;
17617                                 else
17618                                         rack->r_ctl.fsb.hw_tls = 0;
17619                                 KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))),
17620                                         ("rack:%p left_to_send:%u sbavail:%u out:%u",
17621                                         rack, rack->r_ctl.fsb.left_to_send, sbavail(sb),
17622                                          (tp->snd_max - tp->snd_una)));
17623                                 if (rack->r_ctl.fsb.left_to_send < segsiz)
17624                                         rack->r_fast_output = 0;
17625                                 else {
17626                                         if (rack->r_ctl.fsb.left_to_send == (sbavail(sb) - (tp->snd_max - tp->snd_una)))
17627                                                 rack->r_ctl.fsb.rfo_apply_push = 1;
17628                                         else
17629                                                 rack->r_ctl.fsb.rfo_apply_push = 0;
17630                                 }
17631                         } else
17632                                 rack->r_fast_output = 0;
17633
17634
17635                         rack_log_fsb(rack, tp, so, flags,
17636                                      ipoptlen, orig_len, len, 0,
17637                                      1, optlen, __LINE__, 1);
17638                         if (SEQ_GT(tp->snd_max, tp->snd_nxt))
17639                                 tp->snd_nxt = tp->snd_max;
17640                 } else {
17641                         int end_window = 0;
17642                         uint32_t seq = tp->gput_ack;
17643
17644                         rsm = RB_MAX(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
17645                         if (rsm) {
17646                                 /*
17647                                  * Mark the last sent that we just-returned (hinting
17648                                  * that delayed ack may play a role in any rtt measurement).
17649                                  */
17650                                 rsm->r_just_ret = 1;
17651                         }
17652                         counter_u64_add(rack_out_size[TCP_MSS_ACCT_JUSTRET], 1);
17653                         rack->r_ctl.rc_agg_delayed = 0;
17654                         rack->r_early = 0;
17655                         rack->r_late = 0;
17656                         rack->r_ctl.rc_agg_early = 0;
17657                         if ((ctf_outstanding(tp) +
17658                              min(max(segsiz, (rack->r_ctl.rc_high_rwnd/2)),
17659                                  minseg)) >= tp->snd_wnd) {
17660                                 /* We are limited by the rwnd */
17661                                 app_limited = CTF_JR_RWND_LIMITED;
17662                                 if (IN_FASTRECOVERY(tp->t_flags))
17663                                     rack->r_ctl.rc_prr_sndcnt = 0;
17664                         } else if (ctf_outstanding(tp) >= sbavail(sb)) {
17665                                 /* We are limited by whats available -- app limited */
17666                                 app_limited = CTF_JR_APP_LIMITED;
17667                                 if (IN_FASTRECOVERY(tp->t_flags))
17668                                     rack->r_ctl.rc_prr_sndcnt = 0;
17669                         } else if ((idle == 0) &&
17670                                    ((tp->t_flags & TF_NODELAY) == 0) &&
17671                                    ((uint32_t)len + (uint32_t)sb_offset >= sbavail(sb)) &&
17672                                    (len < segsiz)) {
17673                                 /*
17674                                  * No delay is not on and the
17675                                  * user is sending less than 1MSS. This
17676                                  * brings out SWS avoidance so we
17677                                  * don't send. Another app-limited case.
17678                                  */
17679                                 app_limited = CTF_JR_APP_LIMITED;
17680                         } else if (tp->t_flags & TF_NOPUSH) {
17681                                 /*
17682                                  * The user has requested no push of
17683                                  * the last segment and we are
17684                                  * at the last segment. Another app
17685                                  * limited case.
17686                                  */
17687                                 app_limited = CTF_JR_APP_LIMITED;
17688                         } else if ((ctf_outstanding(tp) + minseg) > cwnd_to_use) {
17689                                 /* Its the cwnd */
17690                                 app_limited = CTF_JR_CWND_LIMITED;
17691                         } else if (IN_FASTRECOVERY(tp->t_flags) &&
17692                                    (rack->rack_no_prr == 0) &&
17693                                    (rack->r_ctl.rc_prr_sndcnt < segsiz)) {
17694                                 app_limited = CTF_JR_PRR;
17695                         } else {
17696                                 /* Now why here are we not sending? */
17697 #ifdef NOW
17698 #ifdef INVARIANTS
17699                                 panic("rack:%p hit JR_ASSESSING case cwnd_to_use:%u?", rack, cwnd_to_use);
17700 #endif
17701 #endif
17702                                 app_limited = CTF_JR_ASSESSING;
17703                         }
17704                         /*
17705                          * App limited in some fashion, for our pacing GP
17706                          * measurements we don't want any gap (even cwnd).
17707                          * Close  down the measurement window.
17708                          */
17709                         if (rack_cwnd_block_ends_measure &&
17710                             ((app_limited == CTF_JR_CWND_LIMITED) ||
17711                              (app_limited == CTF_JR_PRR))) {
17712                                 /*
17713                                  * The reason we are not sending is
17714                                  * the cwnd (or prr). We have been configured
17715                                  * to end the measurement window in
17716                                  * this case.
17717                                  */
17718                                 end_window = 1;
17719                         } else if (rack_rwnd_block_ends_measure &&
17720                                    (app_limited == CTF_JR_RWND_LIMITED)) {
17721                                 /*
17722                                  * We are rwnd limited and have been
17723                                  * configured to end the measurement
17724                                  * window in this case.
17725                                  */
17726                                 end_window = 1;
17727                         } else if (app_limited == CTF_JR_APP_LIMITED) {
17728                                 /*
17729                                  * A true application limited period, we have
17730                                  * ran out of data.
17731                                  */
17732                                 end_window = 1;
17733                         } else if (app_limited == CTF_JR_ASSESSING) {
17734                                 /*
17735                                  * In the assessing case we hit the end of
17736                                  * the if/else and had no known reason
17737                                  * This will panic us under invariants..
17738                                  *
17739                                  * If we get this out in logs we need to
17740                                  * investagate which reason we missed.
17741                                  */
17742                                 end_window = 1;
17743                         }
17744                         if (end_window) {
17745                                 uint8_t log = 0;
17746
17747                                 /* Adjust the Gput measurement */
17748                                 if ((tp->t_flags & TF_GPUTINPROG) &&
17749                                     SEQ_GT(tp->gput_ack, tp->snd_max)) {
17750                                         tp->gput_ack = tp->snd_max;
17751                                         if ((tp->gput_ack - tp->gput_seq) < (MIN_GP_WIN * segsiz)) {
17752                                                 /*
17753                                                  * There is not enough to measure.
17754                                                  */
17755                                                 tp->t_flags &= ~TF_GPUTINPROG;
17756                                                 rack_log_pacing_delay_calc(rack, (tp->gput_ack - tp->gput_seq) /*flex2*/,
17757                                                                            rack->r_ctl.rc_gp_srtt /*flex1*/,
17758                                                                            tp->gput_seq,
17759                                                                            0, 0, 18, __LINE__, NULL, 0);
17760                                         } else
17761                                                 log = 1;
17762                                 }
17763                                 /* Mark the last packet has app limited */
17764                                 rsm = RB_MAX(rack_rb_tree_head, &rack->r_ctl.rc_mtree);
17765                                 if (rsm && ((rsm->r_flags & RACK_APP_LIMITED) == 0)) {
17766                                         if (rack->r_ctl.rc_app_limited_cnt == 0)
17767                                                 rack->r_ctl.rc_end_appl = rack->r_ctl.rc_first_appl = rsm;
17768                                         else {
17769                                                 /*
17770                                                  * Go out to the end app limited and mark
17771                                                  * this new one as next and move the end_appl up
17772                                                  * to this guy.
17773                                                  */
17774                                                 if (rack->r_ctl.rc_end_appl)
17775                                                         rack->r_ctl.rc_end_appl->r_nseq_appl = rsm->r_start;
17776                                                 rack->r_ctl.rc_end_appl = rsm;
17777                                         }
17778                                         rsm->r_flags |= RACK_APP_LIMITED;
17779                                         rack->r_ctl.rc_app_limited_cnt++;
17780                                 }
17781                                 if (log)
17782                                         rack_log_pacing_delay_calc(rack,
17783                                                                    rack->r_ctl.rc_app_limited_cnt, seq,
17784                                                                    tp->gput_ack, 0, 0, 4, __LINE__, NULL, 0);
17785                         }
17786                 }
17787                 /* Check if we need to go into persists or not */
17788                 if ((tp->snd_max == tp->snd_una) &&
17789                     TCPS_HAVEESTABLISHED(tp->t_state) &&
17790                     sbavail(sb) &&
17791                     (sbavail(sb) > tp->snd_wnd) &&
17792                     (tp->snd_wnd < min((rack->r_ctl.rc_high_rwnd/2), minseg))) {
17793                         /* Yes lets make sure to move to persist before timer-start */
17794                         rack_enter_persist(tp, rack, rack->r_ctl.rc_rcvtime);
17795                 }
17796                 rack_start_hpts_timer(rack, tp, cts, slot, tot_len_this_send, sup_rack);
17797                 rack_log_type_just_return(rack, cts, tot_len_this_send, slot, hpts_calling, app_limited, cwnd_to_use);
17798         }
17799 #ifdef NETFLIX_SHARED_CWND
17800         if ((sbavail(sb) == 0) &&
17801             rack->r_ctl.rc_scw) {
17802                 tcp_shared_cwnd_idle(rack->r_ctl.rc_scw, rack->r_ctl.rc_scw_index);
17803                 rack->rack_scwnd_is_idle = 1;
17804         }
17805 #endif
17806 #ifdef TCP_ACCOUNTING
17807         if (tot_len_this_send > 0) {
17808                 crtsc = get_cyclecount();
17809                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
17810                         tp->tcp_cnt_counters[SND_OUT_DATA]++;
17811                 }
17812                 counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], 1);
17813                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
17814                         tp->tcp_proc_time[SND_OUT_DATA] += (crtsc - ts_val);
17815                 }
17816                 counter_u64_add(tcp_proc_time[SND_OUT_DATA], (crtsc - ts_val));
17817                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
17818                         tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((tot_len_this_send + segsiz - 1) / segsiz);
17819                 }
17820                 counter_u64_add(tcp_cnt_counters[CNT_OF_MSS_OUT], ((tot_len_this_send + segsiz - 1) / segsiz));
17821         } else {
17822                 crtsc = get_cyclecount();
17823                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
17824                         tp->tcp_cnt_counters[SND_LIMITED]++;
17825                 }
17826                 counter_u64_add(tcp_cnt_counters[SND_LIMITED], 1);
17827                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
17828                         tp->tcp_proc_time[SND_LIMITED] += (crtsc - ts_val);
17829                 }
17830                 counter_u64_add(tcp_proc_time[SND_LIMITED], (crtsc - ts_val));
17831         }
17832         sched_unpin();
17833 #endif
17834         return (0);
17835
17836 send:
17837         if (rsm || sack_rxmit)
17838                 counter_u64_add(rack_nfto_resend, 1);
17839         else
17840                 counter_u64_add(rack_non_fto_send, 1);
17841         if ((flags & TH_FIN) &&
17842             sbavail(sb)) {
17843                 /*
17844                  * We do not transmit a FIN
17845                  * with data outstanding. We
17846                  * need to make it so all data
17847                  * is acked first.
17848                  */
17849                 flags &= ~TH_FIN;
17850         }
17851         /* Enforce stack imposed max seg size if we have one */
17852         if (rack->r_ctl.rc_pace_max_segs &&
17853             (len > rack->r_ctl.rc_pace_max_segs)) {
17854                 mark = 1;
17855                 len = rack->r_ctl.rc_pace_max_segs;
17856         }
17857         SOCKBUF_LOCK_ASSERT(sb);
17858         if (len > 0) {
17859                 if (len >= segsiz)
17860                         tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;
17861                 else
17862                         tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;
17863         }
17864         /*
17865          * Before ESTABLISHED, force sending of initial options unless TCP
17866          * set not to do any options. NOTE: we assume that the IP/TCP header
17867          * plus TCP options always fit in a single mbuf, leaving room for a
17868          * maximum link header, i.e. max_linkhdr + sizeof (struct tcpiphdr)
17869          * + optlen <= MCLBYTES
17870          */
17871         optlen = 0;
17872 #ifdef INET6
17873         if (isipv6)
17874                 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
17875         else
17876 #endif
17877                 hdrlen = sizeof(struct tcpiphdr);
17878
17879         /*
17880          * Compute options for segment. We only have to care about SYN and
17881          * established connection segments.  Options for SYN-ACK segments
17882          * are handled in TCP syncache.
17883          */
17884         to.to_flags = 0;
17885         if ((tp->t_flags & TF_NOOPT) == 0) {
17886                 /* Maximum segment size. */
17887                 if (flags & TH_SYN) {
17888                         tp->snd_nxt = tp->iss;
17889                         to.to_mss = tcp_mssopt(&inp->inp_inc);
17890                         if (tp->t_port)
17891                                 to.to_mss -= V_tcp_udp_tunneling_overhead;
17892                         to.to_flags |= TOF_MSS;
17893
17894                         /*
17895                          * On SYN or SYN|ACK transmits on TFO connections,
17896                          * only include the TFO option if it is not a
17897                          * retransmit, as the presence of the TFO option may
17898                          * have caused the original SYN or SYN|ACK to have
17899                          * been dropped by a middlebox.
17900                          */
17901                         if (IS_FASTOPEN(tp->t_flags) &&
17902                             (tp->t_rxtshift == 0)) {
17903                                 if (tp->t_state == TCPS_SYN_RECEIVED) {
17904                                         to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN;
17905                                         to.to_tfo_cookie =
17906                                                 (u_int8_t *)&tp->t_tfo_cookie.server;
17907                                         to.to_flags |= TOF_FASTOPEN;
17908                                         wanted_cookie = 1;
17909                                 } else if (tp->t_state == TCPS_SYN_SENT) {
17910                                         to.to_tfo_len =
17911                                                 tp->t_tfo_client_cookie_len;
17912                                         to.to_tfo_cookie =
17913                                                 tp->t_tfo_cookie.client;
17914                                         to.to_flags |= TOF_FASTOPEN;
17915                                         wanted_cookie = 1;
17916                                         /*
17917                                          * If we wind up having more data to
17918                                          * send with the SYN than can fit in
17919                                          * one segment, don't send any more
17920                                          * until the SYN|ACK comes back from
17921                                          * the other end.
17922                                          */
17923                                         sendalot = 0;
17924                                 }
17925                         }
17926                 }
17927                 /* Window scaling. */
17928                 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
17929                         to.to_wscale = tp->request_r_scale;
17930                         to.to_flags |= TOF_SCALE;
17931                 }
17932                 /* Timestamps. */
17933                 if ((tp->t_flags & TF_RCVD_TSTMP) ||
17934                     ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {
17935                         to.to_tsval = ms_cts + tp->ts_offset;
17936                         to.to_tsecr = tp->ts_recent;
17937                         to.to_flags |= TOF_TS;
17938                 }
17939                 /* Set receive buffer autosizing timestamp. */
17940                 if (tp->rfbuf_ts == 0 &&
17941                     (so->so_rcv.sb_flags & SB_AUTOSIZE))
17942                         tp->rfbuf_ts = tcp_ts_getticks();
17943                 /* Selective ACK's. */
17944                 if (tp->t_flags & TF_SACK_PERMIT) {
17945                         if (flags & TH_SYN)
17946                                 to.to_flags |= TOF_SACKPERM;
17947                         else if (TCPS_HAVEESTABLISHED(tp->t_state) &&
17948                                  tp->rcv_numsacks > 0) {
17949                                 to.to_flags |= TOF_SACK;
17950                                 to.to_nsacks = tp->rcv_numsacks;
17951                                 to.to_sacks = (u_char *)tp->sackblks;
17952                         }
17953                 }
17954 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
17955                 /* TCP-MD5 (RFC2385). */
17956                 if (tp->t_flags & TF_SIGNATURE)
17957                         to.to_flags |= TOF_SIGNATURE;
17958 #endif                          /* TCP_SIGNATURE */
17959
17960                 /* Processing the options. */
17961                 hdrlen += optlen = tcp_addoptions(&to, opt);
17962                 /*
17963                  * If we wanted a TFO option to be added, but it was unable
17964                  * to fit, ensure no data is sent.
17965                  */
17966                 if (IS_FASTOPEN(tp->t_flags) && wanted_cookie &&
17967                     !(to.to_flags & TOF_FASTOPEN))
17968                         len = 0;
17969         }
17970         if (tp->t_port) {
17971                 if (V_tcp_udp_tunneling_port == 0) {
17972                         /* The port was removed?? */
17973                         SOCKBUF_UNLOCK(&so->so_snd);
17974 #ifdef TCP_ACCOUNTING
17975                         crtsc = get_cyclecount();
17976                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
17977                                 tp->tcp_cnt_counters[SND_OUT_FAIL]++;
17978                         }
17979                         counter_u64_add(tcp_cnt_counters[SND_OUT_FAIL], 1);
17980                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
17981                                 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val);
17982                         }
17983                         counter_u64_add(tcp_proc_time[SND_OUT_FAIL], (crtsc - ts_val));
17984                         sched_unpin();
17985 #endif
17986                         return (EHOSTUNREACH);
17987                 }
17988                 hdrlen += sizeof(struct udphdr);
17989         }
17990 #ifdef INET6
17991         if (isipv6)
17992                 ipoptlen = ip6_optlen(tp->t_inpcb);
17993         else
17994 #endif
17995                 if (tp->t_inpcb->inp_options)
17996                         ipoptlen = tp->t_inpcb->inp_options->m_len -
17997                                 offsetof(struct ipoption, ipopt_list);
17998                 else
17999                         ipoptlen = 0;
18000 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
18001         ipoptlen += ipsec_optlen;
18002 #endif
18003
18004         /*
18005          * Adjust data length if insertion of options will bump the packet
18006          * length beyond the t_maxseg length. Clear the FIN bit because we
18007          * cut off the tail of the segment.
18008          */
18009         if (len + optlen + ipoptlen > tp->t_maxseg) {
18010                 if (tso) {
18011                         uint32_t if_hw_tsomax;
18012                         uint32_t moff;
18013                         int32_t max_len;
18014
18015                         /* extract TSO information */
18016                         if_hw_tsomax = tp->t_tsomax;
18017                         if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
18018                         if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
18019                         KASSERT(ipoptlen == 0,
18020                                 ("%s: TSO can't do IP options", __func__));
18021
18022                         /*
18023                          * Check if we should limit by maximum payload
18024                          * length:
18025                          */
18026                         if (if_hw_tsomax != 0) {
18027                                 /* compute maximum TSO length */
18028                                 max_len = (if_hw_tsomax - hdrlen -
18029                                            max_linkhdr);
18030                                 if (max_len <= 0) {
18031                                         len = 0;
18032                                 } else if (len > max_len) {
18033                                         sendalot = 1;
18034                                         len = max_len;
18035                                         mark = 2;
18036                                 }
18037                         }
18038                         /*
18039                          * Prevent the last segment from being fractional
18040                          * unless the send sockbuf can be emptied:
18041                          */
18042                         max_len = (tp->t_maxseg - optlen);
18043                         if ((sb_offset + len) < sbavail(sb)) {
18044                                 moff = len % (u_int)max_len;
18045                                 if (moff != 0) {
18046                                         mark = 3;
18047                                         len -= moff;
18048                                 }
18049                         }
18050                         /*
18051                          * In case there are too many small fragments don't
18052                          * use TSO:
18053                          */
18054                         if (len <= segsiz) {
18055                                 mark = 4;
18056                                 tso = 0;
18057                         }
18058                         /*
18059                          * Send the FIN in a separate segment after the bulk
18060                          * sending is done. We don't trust the TSO
18061                          * implementations to clear the FIN flag on all but
18062                          * the last segment.
18063                          */
18064                         if (tp->t_flags & TF_NEEDFIN) {
18065                                 sendalot = 4;
18066                         }
18067                 } else {
18068                         mark = 5;
18069                         if (optlen + ipoptlen >= tp->t_maxseg) {
18070                                 /*
18071                                  * Since we don't have enough space to put
18072                                  * the IP header chain and the TCP header in
18073                                  * one packet as required by RFC 7112, don't
18074                                  * send it. Also ensure that at least one
18075                                  * byte of the payload can be put into the
18076                                  * TCP segment.
18077                                  */
18078                                 SOCKBUF_UNLOCK(&so->so_snd);
18079                                 error = EMSGSIZE;
18080                                 sack_rxmit = 0;
18081                                 goto out;
18082                         }
18083                         len = tp->t_maxseg - optlen - ipoptlen;
18084                         sendalot = 5;
18085                 }
18086         } else {
18087                 tso = 0;
18088                 mark = 6;
18089         }
18090         KASSERT(len + hdrlen + ipoptlen <= IP_MAXPACKET,
18091                 ("%s: len > IP_MAXPACKET", __func__));
18092 #ifdef DIAGNOSTIC
18093 #ifdef INET6
18094         if (max_linkhdr + hdrlen > MCLBYTES)
18095 #else
18096                 if (max_linkhdr + hdrlen > MHLEN)
18097 #endif
18098                         panic("tcphdr too big");
18099 #endif
18100
18101         /*
18102          * This KASSERT is here to catch edge cases at a well defined place.
18103          * Before, those had triggered (random) panic conditions further
18104          * down.
18105          */
18106         KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
18107         if ((len == 0) &&
18108             (flags & TH_FIN) &&
18109             (sbused(sb))) {
18110                 /*
18111                  * We have outstanding data, don't send a fin by itself!.
18112                  */
18113                 goto just_return;
18114         }
18115         /*
18116          * Grab a header mbuf, attaching a copy of data to be transmitted,
18117          * and initialize the header from the template for sends on this
18118          * connection.
18119          */
18120         hw_tls = (sb->sb_flags & SB_TLS_IFNET) != 0;
18121         if (len) {
18122                 uint32_t max_val;
18123                 uint32_t moff;
18124
18125                 if (rack->r_ctl.rc_pace_max_segs)
18126                         max_val = rack->r_ctl.rc_pace_max_segs;
18127                 else if (rack->rc_user_set_max_segs)
18128                         max_val = rack->rc_user_set_max_segs * segsiz;
18129                 else
18130                         max_val = len;
18131                 /*
18132                  * We allow a limit on sending with hptsi.
18133                  */
18134                 if (len > max_val) {
18135                         mark = 7;
18136                         len = max_val;
18137                 }
18138 #ifdef INET6
18139                 if (MHLEN < hdrlen + max_linkhdr)
18140                         m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
18141                 else
18142 #endif
18143                         m = m_gethdr(M_NOWAIT, MT_DATA);
18144
18145                 if (m == NULL) {
18146                         SOCKBUF_UNLOCK(sb);
18147                         error = ENOBUFS;
18148                         sack_rxmit = 0;
18149                         goto out;
18150                 }
18151                 m->m_data += max_linkhdr;
18152                 m->m_len = hdrlen;
18153
18154                 /*
18155                  * Start the m_copy functions from the closest mbuf to the
18156                  * sb_offset in the socket buffer chain.
18157                  */
18158                 mb = sbsndptr_noadv(sb, sb_offset, &moff);
18159                 s_mb = mb;
18160                 s_moff = moff;
18161                 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) {
18162                         m_copydata(mb, moff, (int)len,
18163                                    mtod(m, caddr_t)+hdrlen);
18164                         if (SEQ_LT(tp->snd_nxt, tp->snd_max))
18165                                 sbsndptr_adv(sb, mb, len);
18166                         m->m_len += len;
18167                 } else {
18168                         struct sockbuf *msb;
18169
18170                         if (SEQ_LT(tp->snd_nxt, tp->snd_max))
18171                                 msb = NULL;
18172                         else
18173                                 msb = sb;
18174                         m->m_next = tcp_m_copym(
18175                                 mb, moff, &len,
18176                                 if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, msb,
18177                                 ((rsm == NULL) ? hw_tls : 0)
18178 #ifdef NETFLIX_COPY_ARGS
18179                                 , &filled_all
18180 #endif
18181                                 );
18182                         if (len <= (tp->t_maxseg - optlen)) {
18183                                 /*
18184                                  * Must have ran out of mbufs for the copy
18185                                  * shorten it to no longer need tso. Lets
18186                                  * not put on sendalot since we are low on
18187                                  * mbufs.
18188                                  */
18189                                 tso = 0;
18190                         }
18191                         if (m->m_next == NULL) {
18192                                 SOCKBUF_UNLOCK(sb);
18193                                 (void)m_free(m);
18194                                 error = ENOBUFS;
18195                                 sack_rxmit = 0;
18196                                 goto out;
18197                         }
18198                 }
18199                 if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
18200                         if (rsm && (rsm->r_flags & RACK_TLP)) {
18201                                 /*
18202                                  * TLP should not count in retran count, but
18203                                  * in its own bin
18204                                  */
18205                                 counter_u64_add(rack_tlp_retran, 1);
18206                                 counter_u64_add(rack_tlp_retran_bytes, len);
18207                         } else {
18208                                 tp->t_sndrexmitpack++;
18209                                 KMOD_TCPSTAT_INC(tcps_sndrexmitpack);
18210                                 KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len);
18211                         }
18212 #ifdef STATS
18213                         stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
18214                                                  len);
18215 #endif
18216                 } else {
18217                         KMOD_TCPSTAT_INC(tcps_sndpack);
18218                         KMOD_TCPSTAT_ADD(tcps_sndbyte, len);
18219 #ifdef STATS
18220                         stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
18221                                                  len);
18222 #endif
18223                 }
18224                 /*
18225                  * If we're sending everything we've got, set PUSH. (This
18226                  * will keep happy those implementations which only give
18227                  * data to the user when a buffer fills or a PUSH comes in.)
18228                  */
18229                 if (sb_offset + len == sbused(sb) &&
18230                     sbused(sb) &&
18231                     !(flags & TH_SYN)) {
18232                         flags |= TH_PUSH;
18233                         add_flag |= RACK_HAD_PUSH;
18234                 }
18235
18236                 SOCKBUF_UNLOCK(sb);
18237         } else {
18238                 SOCKBUF_UNLOCK(sb);
18239                 if (tp->t_flags & TF_ACKNOW)
18240                         KMOD_TCPSTAT_INC(tcps_sndacks);
18241                 else if (flags & (TH_SYN | TH_FIN | TH_RST))
18242                         KMOD_TCPSTAT_INC(tcps_sndctrl);
18243                 else
18244                         KMOD_TCPSTAT_INC(tcps_sndwinup);
18245
18246                 m = m_gethdr(M_NOWAIT, MT_DATA);
18247                 if (m == NULL) {
18248                         error = ENOBUFS;
18249                         sack_rxmit = 0;
18250                         goto out;
18251                 }
18252 #ifdef INET6
18253                 if (isipv6 && (MHLEN < hdrlen + max_linkhdr) &&
18254                     MHLEN >= hdrlen) {
18255                         M_ALIGN(m, hdrlen);
18256                 } else
18257 #endif
18258                         m->m_data += max_linkhdr;
18259                 m->m_len = hdrlen;
18260         }
18261         SOCKBUF_UNLOCK_ASSERT(sb);
18262         m->m_pkthdr.rcvif = (struct ifnet *)0;
18263 #ifdef MAC
18264         mac_inpcb_create_mbuf(inp, m);
18265 #endif
18266         if ((ipoptlen == 0) && (rack->r_ctl.fsb.tcp_ip_hdr) &&  rack->r_fsb_inited) {
18267 #ifdef INET6
18268                 if (isipv6)
18269                         ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr;
18270                 else
18271 #endif                          /* INET6 */
18272                         ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
18273                 th = rack->r_ctl.fsb.th;
18274                 udp = rack->r_ctl.fsb.udp;
18275                 if (udp) {
18276 #ifdef INET6
18277                         if (isipv6)
18278                                 ulen = hdrlen + len - sizeof(struct ip6_hdr);
18279                         else
18280 #endif                          /* INET6 */
18281                                 ulen = hdrlen + len - sizeof(struct ip);
18282                         udp->uh_ulen = htons(ulen);
18283                 }
18284         } else {
18285 #ifdef INET6
18286                 if (isipv6) {
18287                         ip6 = mtod(m, struct ip6_hdr *);
18288                         if (tp->t_port) {
18289                                 udp = (struct udphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
18290                                 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
18291                                 udp->uh_dport = tp->t_port;
18292                                 ulen = hdrlen + len - sizeof(struct ip6_hdr);
18293                                 udp->uh_ulen = htons(ulen);
18294                                 th = (struct tcphdr *)(udp + 1);
18295                         } else
18296                                 th = (struct tcphdr *)(ip6 + 1);
18297                         tcpip_fillheaders(inp, tp->t_port, ip6, th);
18298                 } else
18299 #endif                          /* INET6 */
18300                 {
18301                         ip = mtod(m, struct ip *);
18302                         if (tp->t_port) {
18303                                 udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip));
18304                                 udp->uh_sport = htons(V_tcp_udp_tunneling_port);
18305                                 udp->uh_dport = tp->t_port;
18306                                 ulen = hdrlen + len - sizeof(struct ip);
18307                                 udp->uh_ulen = htons(ulen);
18308                                 th = (struct tcphdr *)(udp + 1);
18309                         } else
18310                                 th = (struct tcphdr *)(ip + 1);
18311                         tcpip_fillheaders(inp, tp->t_port, ip, th);
18312                 }
18313         }
18314         /*
18315          * Fill in fields, remembering maximum advertised window for use in
18316          * delaying messages about window sizes. If resending a FIN, be sure
18317          * not to use a new sequence number.
18318          */
18319         if (flags & TH_FIN && tp->t_flags & TF_SENTFIN &&
18320             tp->snd_nxt == tp->snd_max)
18321                 tp->snd_nxt--;
18322         /*
18323          * If we are starting a connection, send ECN setup SYN packet. If we
18324          * are on a retransmit, we may resend those bits a number of times
18325          * as per RFC 3168.
18326          */
18327         if (tp->t_state == TCPS_SYN_SENT && V_tcp_do_ecn) {
18328                 flags |= tcp_ecn_output_syn_sent(tp);
18329         }
18330         /* Also handle parallel SYN for ECN */
18331         if (TCPS_HAVERCVDSYN(tp->t_state) &&
18332             (tp->t_flags2 & TF2_ECN_PERMIT)) {
18333                 int ect = tcp_ecn_output_established(tp, &flags, len, sack_rxmit);
18334                 if ((tp->t_state == TCPS_SYN_RECEIVED) &&
18335                     (tp->t_flags2 & TF2_ECN_SND_ECE))
18336                         tp->t_flags2 &= ~TF2_ECN_SND_ECE;
18337 #ifdef INET6
18338                 if (isipv6) {
18339                         ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << 20);
18340                         ip6->ip6_flow |= htonl(ect << 20);
18341                 }
18342                 else
18343 #endif
18344                 {
18345                         ip->ip_tos &= ~IPTOS_ECN_MASK;
18346                         ip->ip_tos |= ect;
18347                 }
18348         }
18349         /*
18350          * If we are doing retransmissions, then snd_nxt will not reflect
18351          * the first unsent octet.  For ACK only packets, we do not want the
18352          * sequence number of the retransmitted packet, we want the sequence
18353          * number of the next unsent octet.  So, if there is no data (and no
18354          * SYN or FIN), use snd_max instead of snd_nxt when filling in
18355          * ti_seq.  But if we are in persist state, snd_max might reflect
18356          * one byte beyond the right edge of the window, so use snd_nxt in
18357          * that case, since we know we aren't doing a retransmission.
18358          * (retransmit and persist are mutually exclusive...)
18359          */
18360         if (sack_rxmit == 0) {
18361                 if (len || (flags & (TH_SYN | TH_FIN))) {
18362                         th->th_seq = htonl(tp->snd_nxt);
18363                         rack_seq = tp->snd_nxt;
18364                 } else {
18365                         th->th_seq = htonl(tp->snd_max);
18366                         rack_seq = tp->snd_max;
18367                 }
18368         } else {
18369                 th->th_seq = htonl(rsm->r_start);
18370                 rack_seq = rsm->r_start;
18371         }
18372         th->th_ack = htonl(tp->rcv_nxt);
18373         tcp_set_flags(th, flags);
18374         /*
18375          * Calculate receive window.  Don't shrink window, but avoid silly
18376          * window syndrome.
18377          * If a RST segment is sent, advertise a window of zero.
18378          */
18379         if (flags & TH_RST) {
18380                 recwin = 0;
18381         } else {
18382                 if (recwin < (long)(so->so_rcv.sb_hiwat / 4) &&
18383                     recwin < (long)segsiz) {
18384                         recwin = 0;
18385                 }
18386                 if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&
18387                     recwin < (long)(tp->rcv_adv - tp->rcv_nxt))
18388                         recwin = (long)(tp->rcv_adv - tp->rcv_nxt);
18389         }
18390
18391         /*
18392          * According to RFC1323 the window field in a SYN (i.e., a <SYN> or
18393          * <SYN,ACK>) segment itself is never scaled.  The <SYN,ACK> case is
18394          * handled in syncache.
18395          */
18396         if (flags & TH_SYN)
18397                 th->th_win = htons((u_short)
18398                                    (min(sbspace(&so->so_rcv), TCP_MAXWIN)));
18399         else {
18400                 /* Avoid shrinking window with window scaling. */
18401                 recwin = roundup2(recwin, 1 << tp->rcv_scale);
18402                 th->th_win = htons((u_short)(recwin >> tp->rcv_scale));
18403         }
18404         /*
18405          * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0
18406          * window.  This may cause the remote transmitter to stall.  This
18407          * flag tells soreceive() to disable delayed acknowledgements when
18408          * draining the buffer.  This can occur if the receiver is
18409          * attempting to read more data than can be buffered prior to
18410          * transmitting on the connection.
18411          */
18412         if (th->th_win == 0) {
18413                 tp->t_sndzerowin++;
18414                 tp->t_flags |= TF_RXWIN0SENT;
18415         } else
18416                 tp->t_flags &= ~TF_RXWIN0SENT;
18417         tp->snd_up = tp->snd_una;       /* drag it along, its deprecated */
18418         /* Now are we using fsb?, if so copy the template data to the mbuf */
18419         if ((ipoptlen == 0) && (rack->r_ctl.fsb.tcp_ip_hdr) && rack->r_fsb_inited) {
18420                 uint8_t *cpto;
18421
18422                 cpto = mtod(m, uint8_t *);
18423                 memcpy(cpto, rack->r_ctl.fsb.tcp_ip_hdr, rack->r_ctl.fsb.tcp_ip_hdr_len);
18424                 /*
18425                  * We have just copied in:
18426                  * IP/IP6
18427                  * <optional udphdr>
18428                  * tcphdr (no options)
18429                  *
18430                  * We need to grab the correct pointers into the mbuf
18431                  * for both the tcp header, and possibly the udp header (if tunneling).
18432                  * We do this by using the offset in the copy buffer and adding it
18433                  * to the mbuf base pointer (cpto).
18434                  */
18435 #ifdef INET6
18436                 if (isipv6)
18437                         ip6 = mtod(m, struct ip6_hdr *);
18438                 else
18439 #endif                          /* INET6 */
18440                         ip = mtod(m, struct ip *);
18441                 th = (struct tcphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.th - rack->r_ctl.fsb.tcp_ip_hdr));
18442                 /* If we have a udp header lets set it into the mbuf as well */
18443                 if (udp)
18444                         udp = (struct udphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.udp - rack->r_ctl.fsb.tcp_ip_hdr));
18445         }
18446 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
18447         if (to.to_flags & TOF_SIGNATURE) {
18448                 /*
18449                  * Calculate MD5 signature and put it into the place
18450                  * determined before.
18451                  * NOTE: since TCP options buffer doesn't point into
18452                  * mbuf's data, calculate offset and use it.
18453                  */
18454                 if (!TCPMD5_ENABLED() || TCPMD5_OUTPUT(m, th,
18455                                                        (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {
18456                         /*
18457                          * Do not send segment if the calculation of MD5
18458                          * digest has failed.
18459                          */
18460                         goto out;
18461                 }
18462         }
18463 #endif
18464         if (optlen) {
18465                 bcopy(opt, th + 1, optlen);
18466                 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
18467         }
18468         /*
18469          * Put TCP length in extended header, and then checksum extended
18470          * header and data.
18471          */
18472         m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
18473 #ifdef INET6
18474         if (isipv6) {
18475                 /*
18476                  * ip6_plen is not need to be filled now, and will be filled
18477                  * in ip6_output.
18478                  */
18479                 if (tp->t_port) {
18480                         m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
18481                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
18482                         udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
18483                         th->th_sum = htons(0);
18484                         UDPSTAT_INC(udps_opackets);
18485                 } else {
18486                         m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
18487                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
18488                         th->th_sum = in6_cksum_pseudo(ip6,
18489                                                       sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP,
18490                                                       0);
18491                 }
18492         }
18493 #endif
18494 #if defined(INET6) && defined(INET)
18495         else
18496 #endif
18497 #ifdef INET
18498         {
18499                 if (tp->t_port) {
18500                         m->m_pkthdr.csum_flags = CSUM_UDP;
18501                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
18502                         udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
18503                                                 ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
18504                         th->th_sum = htons(0);
18505                         UDPSTAT_INC(udps_opackets);
18506                 } else {
18507                         m->m_pkthdr.csum_flags = CSUM_TCP;
18508                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
18509                         th->th_sum = in_pseudo(ip->ip_src.s_addr,
18510                                                ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
18511                                                                         IPPROTO_TCP + len + optlen));
18512                 }
18513                 /* IP version must be set here for ipv4/ipv6 checking later */
18514                 KASSERT(ip->ip_v == IPVERSION,
18515                         ("%s: IP version incorrect: %d", __func__, ip->ip_v));
18516         }
18517 #endif
18518         /*
18519          * Enable TSO and specify the size of the segments. The TCP pseudo
18520          * header checksum is always provided. XXX: Fixme: This is currently
18521          * not the case for IPv6.
18522          */
18523         if (tso) {
18524                 KASSERT(len > tp->t_maxseg - optlen,
18525                         ("%s: len <= tso_segsz", __func__));
18526                 m->m_pkthdr.csum_flags |= CSUM_TSO;
18527                 m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen;
18528         }
18529         KASSERT(len + hdrlen == m_length(m, NULL),
18530                 ("%s: mbuf chain different than expected: %d + %u != %u",
18531                  __func__, len, hdrlen, m_length(m, NULL)));
18532
18533 #ifdef TCP_HHOOK
18534         /* Run HHOOK_TCP_ESTABLISHED_OUT helper hooks. */
18535         hhook_run_tcp_est_out(tp, th, &to, len, tso);
18536 #endif
18537         /* We're getting ready to send; log now. */
18538         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
18539                 union tcp_log_stackspecific log;
18540
18541                 memset(&log.u_bbr, 0, sizeof(log.u_bbr));
18542                 log.u_bbr.inhpts = tcp_in_hpts(rack->rc_inp);
18543                 if (rack->rack_no_prr)
18544                         log.u_bbr.flex1 = 0;
18545                 else
18546                         log.u_bbr.flex1 = rack->r_ctl.rc_prr_sndcnt;
18547                 log.u_bbr.flex2 = rack->r_ctl.rc_pace_min_segs;
18548                 log.u_bbr.flex3 = rack->r_ctl.rc_pace_max_segs;
18549                 log.u_bbr.flex4 = orig_len;
18550                 if (filled_all)
18551                         log.u_bbr.flex5 = 0x80000000;
18552                 else
18553                         log.u_bbr.flex5 = 0;
18554                 /* Save off the early/late values */
18555                 log.u_bbr.flex6 = rack->r_ctl.rc_agg_early;
18556                 log.u_bbr.applimited = rack->r_ctl.rc_agg_delayed;
18557                 log.u_bbr.bw_inuse = rack_get_bw(rack);
18558                 if (rsm || sack_rxmit) {
18559                         if (doing_tlp)
18560                                 log.u_bbr.flex8 = 2;
18561                         else
18562                                 log.u_bbr.flex8 = 1;
18563                 } else {
18564                         if (doing_tlp)
18565                                 log.u_bbr.flex8 = 3;
18566                         else
18567                                 log.u_bbr.flex8 = 0;
18568                 }
18569                 log.u_bbr.pacing_gain = rack_get_output_gain(rack, rsm);
18570                 log.u_bbr.flex7 = mark;
18571                 log.u_bbr.flex7 <<= 8;
18572                 log.u_bbr.flex7 |= pass;
18573                 log.u_bbr.pkts_out = tp->t_maxseg;
18574                 log.u_bbr.timeStamp = cts;
18575                 log.u_bbr.inflight = ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked);
18576                 log.u_bbr.lt_epoch = cwnd_to_use;
18577                 log.u_bbr.delivered = sendalot;
18578                 lgb = tcp_log_event_(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK,
18579                                      len, &log, false, NULL, NULL, 0, &tv);
18580         } else
18581                 lgb = NULL;
18582
18583         /*
18584          * Fill in IP length and desired time to live and send to IP level.
18585          * There should be a better way to handle ttl and tos; we could keep
18586          * them in the template, but need a way to checksum without them.
18587          */
18588         /*
18589          * m->m_pkthdr.len should have been set before cksum calcuration,
18590          * because in6_cksum() need it.
18591          */
18592 #ifdef INET6
18593         if (isipv6) {
18594                 /*
18595                  * we separately set hoplimit for every segment, since the
18596                  * user might want to change the value via setsockopt. Also,
18597                  * desired default hop limit might be changed via Neighbor
18598                  * Discovery.
18599                  */
18600                 rack->r_ctl.fsb.hoplimit = ip6->ip6_hlim = in6_selecthlim(inp, NULL);
18601
18602                 /*
18603                  * Set the packet size here for the benefit of DTrace
18604                  * probes. ip6_output() will set it properly; it's supposed
18605                  * to include the option header lengths as well.
18606                  */
18607                 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
18608
18609                 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss)
18610                         tp->t_flags2 |= TF2_PLPMTU_PMTUD;
18611                 else
18612                         tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
18613
18614                 if (tp->t_state == TCPS_SYN_SENT)
18615                         TCP_PROBE5(connect__request, NULL, tp, ip6, tp, th);
18616
18617                 TCP_PROBE5(send, NULL, tp, ip6, tp, th);
18618                 /* TODO: IPv6 IP6TOS_ECT bit on */
18619                 error = ip6_output(m,
18620 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
18621                                    inp->in6p_outputopts,
18622 #else
18623                                    NULL,
18624 #endif
18625                                    &inp->inp_route6,
18626                                    ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0),
18627                                    NULL, NULL, inp);
18628
18629                 if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL)
18630                         mtu = inp->inp_route6.ro_nh->nh_mtu;
18631         }
18632 #endif                          /* INET6 */
18633 #if defined(INET) && defined(INET6)
18634         else
18635 #endif
18636 #ifdef INET
18637         {
18638                 ip->ip_len = htons(m->m_pkthdr.len);
18639 #ifdef INET6
18640                 if (inp->inp_vflag & INP_IPV6PROTO)
18641                         ip->ip_ttl = in6_selecthlim(inp, NULL);
18642 #endif                          /* INET6 */
18643                 rack->r_ctl.fsb.hoplimit = ip->ip_ttl;
18644                 /*
18645                  * If we do path MTU discovery, then we set DF on every
18646                  * packet. This might not be the best thing to do according
18647                  * to RFC3390 Section 2. However the tcp hostcache migitates
18648                  * the problem so it affects only the first tcp connection
18649                  * with a host.
18650                  *
18651                  * NB: Don't set DF on small MTU/MSS to have a safe
18652                  * fallback.
18653                  */
18654                 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
18655                         tp->t_flags2 |= TF2_PLPMTU_PMTUD;
18656                         if (tp->t_port == 0 || len < V_tcp_minmss) {
18657                                 ip->ip_off |= htons(IP_DF);
18658                         }
18659                 } else {
18660                         tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
18661                 }
18662
18663                 if (tp->t_state == TCPS_SYN_SENT)
18664                         TCP_PROBE5(connect__request, NULL, tp, ip, tp, th);
18665
18666                 TCP_PROBE5(send, NULL, tp, ip, tp, th);
18667
18668                 error = ip_output(m,
18669 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
18670                                   inp->inp_options,
18671 #else
18672                                   NULL,
18673 #endif
18674                                   &inp->inp_route,
18675                                   ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0,
18676                                   inp);
18677                 if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL)
18678                         mtu = inp->inp_route.ro_nh->nh_mtu;
18679         }
18680 #endif                          /* INET */
18681
18682 out:
18683         if (lgb) {
18684                 lgb->tlb_errno = error;
18685                 lgb = NULL;
18686         }
18687         /*
18688          * In transmit state, time the transmission and arrange for the
18689          * retransmit.  In persist state, just set snd_max.
18690          */
18691         if (error == 0) {
18692                 tcp_account_for_send(tp, len, (rsm != NULL), doing_tlp, hw_tls);
18693                 if (rsm && doing_tlp) {
18694                         rack->rc_last_sent_tlp_past_cumack = 0;
18695                         rack->rc_last_sent_tlp_seq_valid = 1;
18696                         rack->r_ctl.last_sent_tlp_seq = rsm->r_start;
18697                         rack->r_ctl.last_sent_tlp_len = rsm->r_end - rsm->r_start;
18698                 }
18699                 rack->forced_ack = 0;   /* If we send something zap the FA flag */
18700                 if (rsm && (doing_tlp == 0)) {
18701                         /* Set we retransmitted */
18702                         rack->rc_gp_saw_rec = 1;
18703                 } else {
18704                         if (cwnd_to_use > tp->snd_ssthresh) {
18705                                 /* Set we sent in CA */
18706                                 rack->rc_gp_saw_ca = 1;
18707                         } else {
18708                                 /* Set we sent in SS */
18709                                 rack->rc_gp_saw_ss = 1;
18710                         }
18711                 }
18712                 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
18713                     (tp->t_flags & TF_SACK_PERMIT) &&
18714                     tp->rcv_numsacks > 0)
18715                         tcp_clean_dsack_blocks(tp);
18716                 tot_len_this_send += len;
18717                 if (len == 0)
18718                         counter_u64_add(rack_out_size[TCP_MSS_ACCT_SNDACK], 1);
18719                 else if (len == 1) {
18720                         counter_u64_add(rack_out_size[TCP_MSS_ACCT_PERSIST], 1);
18721                 } else if (len > 1) {
18722                         int idx;
18723
18724                         idx = (len / segsiz) + 3;
18725                         if (idx >= TCP_MSS_ACCT_ATIMER)
18726                                 counter_u64_add(rack_out_size[(TCP_MSS_ACCT_ATIMER-1)], 1);
18727                         else
18728                                 counter_u64_add(rack_out_size[idx], 1);
18729                 }
18730         }
18731         if ((rack->rack_no_prr == 0) &&
18732             sub_from_prr &&
18733             (error == 0)) {
18734                 if (rack->r_ctl.rc_prr_sndcnt >= len)
18735                         rack->r_ctl.rc_prr_sndcnt -= len;
18736                 else
18737                         rack->r_ctl.rc_prr_sndcnt = 0;
18738         }
18739         sub_from_prr = 0;
18740         if (doing_tlp) {
18741                 /* Make sure the TLP is added */
18742                 add_flag |= RACK_TLP;
18743         } else if (rsm) {
18744                 /* If its a resend without TLP then it must not have the flag */
18745                 rsm->r_flags &= ~RACK_TLP;
18746         }
18747         rack_log_output(tp, &to, len, rack_seq, (uint8_t) flags, error,
18748                         rack_to_usec_ts(&tv),
18749                         rsm, add_flag, s_mb, s_moff, hw_tls);
18750
18751
18752         if ((error == 0) &&
18753             (len > 0) &&
18754             (tp->snd_una == tp->snd_max))
18755                 rack->r_ctl.rc_tlp_rxt_last_time = cts;
18756         {
18757                 tcp_seq startseq = tp->snd_nxt;
18758
18759                 /* Track our lost count */
18760                 if (rsm && (doing_tlp == 0))
18761                         rack->r_ctl.rc_loss_count += rsm->r_end - rsm->r_start;
18762                 /*
18763                  * Advance snd_nxt over sequence space of this segment.
18764                  */
18765                 if (error)
18766                         /* We don't log or do anything with errors */
18767                         goto nomore;
18768                 if (doing_tlp == 0) {
18769                         if (rsm == NULL) {
18770                                 /*
18771                                  * Not a retransmission of some
18772                                  * sort, new data is going out so
18773                                  * clear our TLP count and flag.
18774                                  */
18775                                 rack->rc_tlp_in_progress = 0;
18776                                 rack->r_ctl.rc_tlp_cnt_out = 0;
18777                         }
18778                 } else {
18779                         /*
18780                          * We have just sent a TLP, mark that it is true
18781                          * and make sure our in progress is set so we
18782                          * continue to check the count.
18783                          */
18784                         rack->rc_tlp_in_progress = 1;
18785                         rack->r_ctl.rc_tlp_cnt_out++;
18786                 }
18787                 if (flags & (TH_SYN | TH_FIN)) {
18788                         if (flags & TH_SYN)
18789                                 tp->snd_nxt++;
18790                         if (flags & TH_FIN) {
18791                                 tp->snd_nxt++;
18792                                 tp->t_flags |= TF_SENTFIN;
18793                         }
18794                 }
18795                 /* In the ENOBUFS case we do *not* update snd_max */
18796                 if (sack_rxmit)
18797                         goto nomore;
18798
18799                 tp->snd_nxt += len;
18800                 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
18801                         if (tp->snd_una == tp->snd_max) {
18802                                 /*
18803                                  * Update the time we just added data since
18804                                  * none was outstanding.
18805                                  */
18806                                 rack_log_progress_event(rack, tp, ticks, PROGRESS_START, __LINE__);
18807                                 tp->t_acktime = ticks;
18808                         }
18809                         tp->snd_max = tp->snd_nxt;
18810                         /*
18811                          * Time this transmission if not a retransmission and
18812                          * not currently timing anything.
18813                          * This is only relevant in case of switching back to
18814                          * the base stack.
18815                          */
18816                         if (tp->t_rtttime == 0) {
18817                                 tp->t_rtttime = ticks;
18818                                 tp->t_rtseq = startseq;
18819                                 KMOD_TCPSTAT_INC(tcps_segstimed);
18820                         }
18821                         if (len &&
18822                             ((tp->t_flags & TF_GPUTINPROG) == 0))
18823                                 rack_start_gp_measurement(tp, rack, startseq, sb_offset);
18824                 }
18825                 /*
18826                  * If we are doing FO we need to update the mbuf position and subtract
18827                  * this happens when the peer sends us duplicate information and
18828                  * we thus want to send a DSACK.
18829                  *
18830                  * XXXRRS: This brings to mind a ?, when we send a DSACK block is TSO
18831                  * turned off? If not then we are going to echo multiple DSACK blocks
18832                  * out (with the TSO), which we should not be doing.
18833                  */
18834                 if (rack->r_fast_output && len) {
18835                         if (rack->r_ctl.fsb.left_to_send > len)
18836                                 rack->r_ctl.fsb.left_to_send -= len;
18837                         else
18838                                 rack->r_ctl.fsb.left_to_send = 0;
18839                         if (rack->r_ctl.fsb.left_to_send < segsiz)
18840                                 rack->r_fast_output = 0;
18841                         if (rack->r_fast_output) {
18842                                 rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off);
18843                                 rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len;
18844                         }
18845                 }
18846         }
18847 nomore:
18848         if (error) {
18849                 rack->r_ctl.rc_agg_delayed = 0;
18850                 rack->r_early = 0;
18851                 rack->r_late = 0;
18852                 rack->r_ctl.rc_agg_early = 0;
18853                 SOCKBUF_UNLOCK_ASSERT(sb);      /* Check gotos. */
18854                 /*
18855                  * Failures do not advance the seq counter above. For the
18856                  * case of ENOBUFS we will fall out and retry in 1ms with
18857                  * the hpts. Everything else will just have to retransmit
18858                  * with the timer.
18859                  *
18860                  * In any case, we do not want to loop around for another
18861                  * send without a good reason.
18862                  */
18863                 sendalot = 0;
18864                 switch (error) {
18865                 case EPERM:
18866                         tp->t_softerror = error;
18867 #ifdef TCP_ACCOUNTING
18868                         crtsc = get_cyclecount();
18869                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
18870                                 tp->tcp_cnt_counters[SND_OUT_FAIL]++;
18871                         }
18872                         counter_u64_add(tcp_cnt_counters[SND_OUT_FAIL], 1);
18873                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
18874                                 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val);
18875                         }
18876                         counter_u64_add(tcp_proc_time[SND_OUT_FAIL], (crtsc - ts_val));
18877                         sched_unpin();
18878 #endif
18879                         return (error);
18880                 case ENOBUFS:
18881                         /*
18882                          * Pace us right away to retry in a some
18883                          * time
18884                          */
18885                         if (rack->r_ctl.crte != NULL) {
18886                                 rack_trace_point(rack, RACK_TP_HWENOBUF);
18887                         } else
18888                                 rack_trace_point(rack, RACK_TP_ENOBUF);
18889                         slot = ((1 + rack->rc_enobuf) * HPTS_USEC_IN_MSEC);
18890                         if (rack->rc_enobuf < 0x7f)
18891                                 rack->rc_enobuf++;
18892                         if (slot < (10 * HPTS_USEC_IN_MSEC))
18893                                 slot = 10 * HPTS_USEC_IN_MSEC;
18894                         if (rack->r_ctl.crte != NULL) {
18895                                 counter_u64_add(rack_saw_enobuf_hw, 1);
18896                                 tcp_rl_log_enobuf(rack->r_ctl.crte);
18897                         }
18898                         counter_u64_add(rack_saw_enobuf, 1);
18899                         goto enobufs;
18900                 case EMSGSIZE:
18901                         /*
18902                          * For some reason the interface we used initially
18903                          * to send segments changed to another or lowered
18904                          * its MTU. If TSO was active we either got an
18905                          * interface without TSO capabilits or TSO was
18906                          * turned off. If we obtained mtu from ip_output()
18907                          * then update it and try again.
18908                          */
18909                         if (tso)
18910                                 tp->t_flags &= ~TF_TSO;
18911                         if (mtu != 0) {
18912                                 tcp_mss_update(tp, -1, mtu, NULL, NULL);
18913                                 goto again;
18914                         }
18915                         slot = 10 * HPTS_USEC_IN_MSEC;
18916                         rack_start_hpts_timer(rack, tp, cts, slot, 0, 0);
18917 #ifdef TCP_ACCOUNTING
18918                         crtsc = get_cyclecount();
18919                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
18920                                 tp->tcp_cnt_counters[SND_OUT_FAIL]++;
18921                         }
18922                         counter_u64_add(tcp_cnt_counters[SND_OUT_FAIL], 1);
18923                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
18924                                 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val);
18925                         }
18926                         counter_u64_add(tcp_proc_time[SND_OUT_FAIL], (crtsc - ts_val));
18927                         sched_unpin();
18928 #endif
18929                         return (error);
18930                 case ENETUNREACH:
18931                         counter_u64_add(rack_saw_enetunreach, 1);
18932                 case EHOSTDOWN:
18933                 case EHOSTUNREACH:
18934                 case ENETDOWN:
18935                         if (TCPS_HAVERCVDSYN(tp->t_state)) {
18936                                 tp->t_softerror = error;
18937                         }
18938                         /* FALLTHROUGH */
18939                 default:
18940                         slot = 10 * HPTS_USEC_IN_MSEC;
18941                         rack_start_hpts_timer(rack, tp, cts, slot, 0, 0);
18942 #ifdef TCP_ACCOUNTING
18943                         crtsc = get_cyclecount();
18944                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
18945                                 tp->tcp_cnt_counters[SND_OUT_FAIL]++;
18946                         }
18947                         counter_u64_add(tcp_cnt_counters[SND_OUT_FAIL], 1);
18948                         if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
18949                                 tp->tcp_proc_time[SND_OUT_FAIL] += (crtsc - ts_val);
18950                         }
18951                         counter_u64_add(tcp_proc_time[SND_OUT_FAIL], (crtsc - ts_val));
18952                         sched_unpin();
18953 #endif
18954                         return (error);
18955                 }
18956         } else {
18957                 rack->rc_enobuf = 0;
18958                 if (IN_FASTRECOVERY(tp->t_flags) && rsm)
18959                         rack->r_ctl.retran_during_recovery += len;
18960         }
18961         KMOD_TCPSTAT_INC(tcps_sndtotal);
18962
18963         /*
18964          * Data sent (as far as we can tell). If this advertises a larger
18965          * window than any other segment, then remember the size of the
18966          * advertised window. Any pending ACK has now been sent.
18967          */
18968         if (recwin > 0 && SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
18969                 tp->rcv_adv = tp->rcv_nxt + recwin;
18970
18971         tp->last_ack_sent = tp->rcv_nxt;
18972         tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
18973 enobufs:
18974         if (sendalot) {
18975                 /* Do we need to turn off sendalot? */
18976                 if (rack->r_ctl.rc_pace_max_segs &&
18977                     (tot_len_this_send >= rack->r_ctl.rc_pace_max_segs)) {
18978                         /* We hit our max. */
18979                         sendalot = 0;
18980                 } else if ((rack->rc_user_set_max_segs) &&
18981                            (tot_len_this_send >= (rack->rc_user_set_max_segs * segsiz))) {
18982                         /* We hit the user defined max */
18983                         sendalot = 0;
18984                 }
18985         }
18986         if ((error == 0) && (flags & TH_FIN))
18987                 tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_FIN);
18988         if (flags & TH_RST) {
18989                 /*
18990                  * We don't send again after sending a RST.
18991                  */
18992                 slot = 0;
18993                 sendalot = 0;
18994                 if (error == 0)
18995                         tcp_log_end_status(tp, TCP_EI_STATUS_SERVER_RST);
18996         } else if ((slot == 0) && (sendalot == 0) && tot_len_this_send) {
18997                 /*
18998                  * Get our pacing rate, if an error
18999                  * occurred in sending (ENOBUF) we would
19000                  * hit the else if with slot preset. Other
19001                  * errors return.
19002                  */
19003                 slot = rack_get_pacing_delay(rack, tp, tot_len_this_send, rsm, segsiz);
19004         }
19005         if (rsm &&
19006             (rsm->r_flags & RACK_HAS_SYN) == 0 &&
19007             rack->use_rack_rr) {
19008                 /* Its a retransmit and we use the rack cheat? */
19009                 if ((slot == 0) ||
19010                     (rack->rc_always_pace == 0) ||
19011                     (rack->r_rr_config == 1)) {
19012                         /*
19013                          * We have no pacing set or we
19014                          * are using old-style rack or
19015                          * we are overriden to use the old 1ms pacing.
19016                          */
19017                         slot = rack->r_ctl.rc_min_to;
19018                 }
19019         }
19020         /* We have sent clear the flag */
19021         rack->r_ent_rec_ns = 0;
19022         if (rack->r_must_retran) {
19023                 if (rsm) {
19024                         rack->r_ctl.rc_out_at_rto -= (rsm->r_end - rsm->r_start);
19025                         if (SEQ_GEQ(rsm->r_end, rack->r_ctl.rc_snd_max_at_rto)) {
19026                                 /*
19027                                  * We have retransmitted all.
19028                                  */
19029                                 rack->r_must_retran = 0;
19030                                 rack->r_ctl.rc_out_at_rto = 0;
19031                         }
19032                 } else if (SEQ_GEQ(tp->snd_max, rack->r_ctl.rc_snd_max_at_rto)) {
19033                         /*
19034                          * Sending new data will also kill
19035                          * the loop.
19036                          */
19037                         rack->r_must_retran = 0;
19038                         rack->r_ctl.rc_out_at_rto = 0;
19039                 }
19040         }
19041         rack->r_ctl.fsb.recwin = recwin;
19042         if ((tp->t_flags & (TF_WASCRECOVERY|TF_WASFRECOVERY)) &&
19043             SEQ_GT(tp->snd_max, rack->r_ctl.rc_snd_max_at_rto)) {
19044                 /*
19045                  * We hit an RTO and now have past snd_max at the RTO
19046                  * clear all the WAS flags.
19047                  */
19048                 tp->t_flags &= ~(TF_WASCRECOVERY|TF_WASFRECOVERY);
19049         }
19050         if (slot) {
19051                 /* set the rack tcb into the slot N */
19052                 if ((error == 0) &&
19053                     rack_use_rfo &&
19054                     ((flags & (TH_SYN|TH_FIN)) == 0) &&
19055                     (rsm == NULL) &&
19056                     (tp->snd_nxt == tp->snd_max) &&
19057                     (ipoptlen == 0) &&
19058                     (tp->rcv_numsacks == 0) &&
19059                     rack->r_fsb_inited &&
19060                     TCPS_HAVEESTABLISHED(tp->t_state) &&
19061                     (rack->r_must_retran == 0) &&
19062                     ((tp->t_flags & TF_NEEDFIN) == 0) &&
19063                     (len > 0) && (orig_len > 0) &&
19064                     (orig_len > len) &&
19065                     ((orig_len - len) >= segsiz) &&
19066                     ((optlen == 0) ||
19067                      ((optlen == TCPOLEN_TSTAMP_APPA) && (to.to_flags & TOF_TS)))) {
19068                         /* We can send at least one more MSS using our fsb */
19069
19070                         rack->r_fast_output = 1;
19071                         rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off);
19072                         rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len;
19073                         rack->r_ctl.fsb.tcp_flags = flags;
19074                         rack->r_ctl.fsb.left_to_send = orig_len - len;
19075                         if (hw_tls)
19076                                 rack->r_ctl.fsb.hw_tls = 1;
19077                         else
19078                                 rack->r_ctl.fsb.hw_tls = 0;
19079                         KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))),
19080                                 ("rack:%p left_to_send:%u sbavail:%u out:%u",
19081                                  rack, rack->r_ctl.fsb.left_to_send, sbavail(sb),
19082                                  (tp->snd_max - tp->snd_una)));
19083                         if (rack->r_ctl.fsb.left_to_send < segsiz)
19084                                 rack->r_fast_output = 0;
19085                         else {
19086                                 if (rack->r_ctl.fsb.left_to_send == (sbavail(sb) - (tp->snd_max - tp->snd_una)))
19087                                         rack->r_ctl.fsb.rfo_apply_push = 1;
19088                                 else
19089                                         rack->r_ctl.fsb.rfo_apply_push = 0;
19090                         }
19091                 } else
19092                         rack->r_fast_output = 0;
19093                 rack_log_fsb(rack, tp, so, flags,
19094                              ipoptlen, orig_len, len, error,
19095                              (rsm == NULL), optlen, __LINE__, 2);
19096         } else if (sendalot) {
19097                 int ret;
19098
19099                 sack_rxmit = 0;
19100                 if ((error == 0) &&
19101                     rack_use_rfo &&
19102                     ((flags & (TH_SYN|TH_FIN)) == 0) &&
19103                     (rsm == NULL) &&
19104                     (ipoptlen == 0) &&
19105                     (tp->rcv_numsacks == 0) &&
19106                     (tp->snd_nxt == tp->snd_max) &&
19107                     (rack->r_must_retran == 0) &&
19108                     rack->r_fsb_inited &&
19109                     TCPS_HAVEESTABLISHED(tp->t_state) &&
19110                     ((tp->t_flags & TF_NEEDFIN) == 0) &&
19111                     (len > 0) && (orig_len > 0) &&
19112                     (orig_len > len) &&
19113                     ((orig_len - len) >= segsiz) &&
19114                     ((optlen == 0) ||
19115                      ((optlen == TCPOLEN_TSTAMP_APPA) && (to.to_flags & TOF_TS)))) {
19116                         /* we can use fast_output for more */
19117
19118                         rack->r_fast_output = 1;
19119                         rack->r_ctl.fsb.m = sbsndmbuf(sb, (tp->snd_max - tp->snd_una), &rack->r_ctl.fsb.off);
19120                         rack->r_ctl.fsb.o_m_len = rack->r_ctl.fsb.m->m_len;
19121                         rack->r_ctl.fsb.tcp_flags = flags;
19122                         rack->r_ctl.fsb.left_to_send = orig_len - len;
19123                         if (hw_tls)
19124                                 rack->r_ctl.fsb.hw_tls = 1;
19125                         else
19126                                 rack->r_ctl.fsb.hw_tls = 0;
19127                         KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(sb) - (tp->snd_max - tp->snd_una))),
19128                                 ("rack:%p left_to_send:%u sbavail:%u out:%u",
19129                                  rack, rack->r_ctl.fsb.left_to_send, sbavail(sb),
19130                                  (tp->snd_max - tp->snd_una)));
19131                         if (rack->r_ctl.fsb.left_to_send < segsiz) {
19132                                 rack->r_fast_output = 0;
19133                         }
19134                         if (rack->r_fast_output) {
19135                                 if (rack->r_ctl.fsb.left_to_send == (sbavail(sb) - (tp->snd_max - tp->snd_una)))
19136                                         rack->r_ctl.fsb.rfo_apply_push = 1;
19137                                 else
19138                                         rack->r_ctl.fsb.rfo_apply_push = 0;
19139                                 rack_log_fsb(rack, tp, so, flags,
19140                                              ipoptlen, orig_len, len, error,
19141                                              (rsm == NULL), optlen, __LINE__, 3);
19142                                 error = 0;
19143                                 ret = rack_fast_output(tp, rack, ts_val, cts, ms_cts, &tv, tot_len_this_send, &error);
19144                                 if (ret >= 0)
19145                                         return (ret);
19146                                 else if (error)
19147                                         goto nomore;
19148
19149                         }
19150                 }
19151                 goto again;
19152         }
19153         /* Assure when we leave that snd_nxt will point to top */
19154         if (SEQ_GT(tp->snd_max, tp->snd_nxt))
19155                 tp->snd_nxt = tp->snd_max;
19156         rack_start_hpts_timer(rack, tp, cts, slot, tot_len_this_send, 0);
19157 #ifdef TCP_ACCOUNTING
19158         crtsc = get_cyclecount() - ts_val;
19159         if (tot_len_this_send) {
19160                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
19161                         tp->tcp_cnt_counters[SND_OUT_DATA]++;
19162                 }
19163                 counter_u64_add(tcp_cnt_counters[SND_OUT_DATA], 1);
19164                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
19165                         tp->tcp_proc_time[SND_OUT_DATA] += crtsc;
19166                 }
19167                 counter_u64_add(tcp_proc_time[SND_OUT_DATA], crtsc);
19168                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
19169                         tp->tcp_cnt_counters[CNT_OF_MSS_OUT] += ((tot_len_this_send + segsiz - 1) /segsiz);
19170                 }
19171                 counter_u64_add(tcp_cnt_counters[CNT_OF_MSS_OUT], ((tot_len_this_send + segsiz - 1) /segsiz));
19172         } else {
19173                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
19174                         tp->tcp_cnt_counters[SND_OUT_ACK]++;
19175                 }
19176                 counter_u64_add(tcp_cnt_counters[SND_OUT_ACK], 1);
19177                 if (tp->t_flags2 & TF2_TCP_ACCOUNTING) {
19178                         tp->tcp_proc_time[SND_OUT_ACK] += crtsc;
19179                 }
19180                 counter_u64_add(tcp_proc_time[SND_OUT_ACK], crtsc);
19181         }
19182         sched_unpin();
19183 #endif
19184         if (error == ENOBUFS)
19185                 error = 0;
19186         return (error);
19187 }
19188
19189 static void
19190 rack_update_seg(struct tcp_rack *rack)
19191 {
19192         uint32_t orig_val;
19193
19194         orig_val = rack->r_ctl.rc_pace_max_segs;
19195         rack_set_pace_segments(rack->rc_tp, rack, __LINE__, NULL);
19196         if (orig_val != rack->r_ctl.rc_pace_max_segs)
19197                 rack_log_pacing_delay_calc(rack, 0, 0, orig_val, 0, 0, 15, __LINE__, NULL, 0);
19198 }
19199
19200 static void
19201 rack_mtu_change(struct tcpcb *tp)
19202 {
19203         /*
19204          * The MSS may have changed
19205          */
19206         struct tcp_rack *rack;
19207         struct rack_sendmap *rsm;
19208
19209         rack = (struct tcp_rack *)tp->t_fb_ptr;
19210         if (rack->r_ctl.rc_pace_min_segs != ctf_fixed_maxseg(tp)) {
19211                 /*
19212                  * The MTU has changed we need to resend everything
19213                  * since all we have sent is lost. We first fix
19214                  * up the mtu though.
19215                  */
19216                 rack_set_pace_segments(tp, rack, __LINE__, NULL);
19217                 /* We treat this like a full retransmit timeout without the cwnd adjustment */
19218                 rack_remxt_tmr(tp);
19219                 rack->r_fast_output = 0;
19220                 rack->r_ctl.rc_out_at_rto = ctf_flight_size(tp,
19221                                                 rack->r_ctl.rc_sacked);
19222                 rack->r_ctl.rc_snd_max_at_rto = tp->snd_max;
19223                 rack->r_must_retran = 1;
19224                 /* Mark all inflight to needing to be rxt'd */
19225                 TAILQ_FOREACH(rsm, &rack->r_ctl.rc_tmap, r_tnext) {
19226                         rsm->r_flags |= RACK_MUST_RXT;
19227                 }
19228         }
19229         sack_filter_clear(&rack->r_ctl.rack_sf, tp->snd_una);
19230         /* We don't use snd_nxt to retransmit */
19231         tp->snd_nxt = tp->snd_max;
19232 }
19233
19234 static int
19235 rack_set_profile(struct tcp_rack *rack, int prof)
19236 {
19237         int err = EINVAL;
19238         if (prof == 1) {
19239                 /* pace_always=1 */
19240                 if (rack->rc_always_pace == 0) {
19241                         if (tcp_can_enable_pacing() == 0)
19242                                 return (EBUSY);
19243                 }
19244                 rack->rc_always_pace = 1;
19245                 if (rack->use_fixed_rate || rack->gp_ready)
19246                         rack_set_cc_pacing(rack);
19247                 rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
19248                 rack->rack_attempt_hdwr_pace = 0;
19249                 /* cmpack=1 */
19250                 if (rack_use_cmp_acks)
19251                         rack->r_use_cmp_ack = 1;
19252                 if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state) &&
19253                     rack->r_use_cmp_ack)
19254                         rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
19255                 /* scwnd=1 */
19256                 rack->rack_enable_scwnd = 1;
19257                 /* dynamic=100 */
19258                 rack->rc_gp_dyn_mul = 1;
19259                 /* gp_inc_ca */
19260                 rack->r_ctl.rack_per_of_gp_ca = 100;
19261                 /* rrr_conf=3 */
19262                 rack->r_rr_config = 3;
19263                 /* npush=2 */
19264                 rack->r_ctl.rc_no_push_at_mrtt = 2;
19265                 /* fillcw=1 */
19266                 rack->rc_pace_to_cwnd = 1;
19267                 rack->rc_pace_fill_if_rttin_range = 0;
19268                 rack->rtt_limit_mul = 0;
19269                 /* noprr=1 */
19270                 rack->rack_no_prr = 1;
19271                 /* lscwnd=1 */
19272                 rack->r_limit_scw = 1;
19273                 /* gp_inc_rec */
19274                 rack->r_ctl.rack_per_of_gp_rec = 90;
19275                 err = 0;
19276
19277         } else if (prof == 3) {
19278                 /* Same as profile one execept fill_cw becomes 2 (less aggressive set) */
19279                 /* pace_always=1 */
19280                 if (rack->rc_always_pace == 0) {
19281                         if (tcp_can_enable_pacing() == 0)
19282                                 return (EBUSY);
19283                 }
19284                 rack->rc_always_pace = 1;
19285                 if (rack->use_fixed_rate || rack->gp_ready)
19286                         rack_set_cc_pacing(rack);
19287                 rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
19288                 rack->rack_attempt_hdwr_pace = 0;
19289                 /* cmpack=1 */
19290                 if (rack_use_cmp_acks)
19291                         rack->r_use_cmp_ack = 1;
19292                 if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state) &&
19293                     rack->r_use_cmp_ack)
19294                         rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
19295                 /* scwnd=1 */
19296                 rack->rack_enable_scwnd = 1;
19297                 /* dynamic=100 */
19298                 rack->rc_gp_dyn_mul = 1;
19299                 /* gp_inc_ca */
19300                 rack->r_ctl.rack_per_of_gp_ca = 100;
19301                 /* rrr_conf=3 */
19302                 rack->r_rr_config = 3;
19303                 /* npush=2 */
19304                 rack->r_ctl.rc_no_push_at_mrtt = 2;
19305                 /* fillcw=2 */
19306                 rack->rc_pace_to_cwnd = 1;
19307                 rack->r_fill_less_agg = 1;
19308                 rack->rc_pace_fill_if_rttin_range = 0;
19309                 rack->rtt_limit_mul = 0;
19310                 /* noprr=1 */
19311                 rack->rack_no_prr = 1;
19312                 /* lscwnd=1 */
19313                 rack->r_limit_scw = 1;
19314                 /* gp_inc_rec */
19315                 rack->r_ctl.rack_per_of_gp_rec = 90;
19316                 err = 0;
19317
19318
19319         } else if (prof == 2) {
19320                 /* cmpack=1 */
19321                 if (rack->rc_always_pace == 0) {
19322                         if (tcp_can_enable_pacing() == 0)
19323                                 return (EBUSY);
19324                 }
19325                 rack->rc_always_pace = 1;
19326                 if (rack->use_fixed_rate || rack->gp_ready)
19327                         rack_set_cc_pacing(rack);
19328                 rack->r_use_cmp_ack = 1;
19329                 if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state))
19330                         rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
19331                 /* pace_always=1 */
19332                 rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
19333                 /* scwnd=1 */
19334                 rack->rack_enable_scwnd = 1;
19335                 /* dynamic=100 */
19336                 rack->rc_gp_dyn_mul = 1;
19337                 rack->r_ctl.rack_per_of_gp_ca = 100;
19338                 /* rrr_conf=3 */
19339                 rack->r_rr_config = 3;
19340                 /* npush=2 */
19341                 rack->r_ctl.rc_no_push_at_mrtt = 2;
19342                 /* fillcw=1 */
19343                 rack->rc_pace_to_cwnd = 1;
19344                 rack->rc_pace_fill_if_rttin_range = 0;
19345                 rack->rtt_limit_mul = 0;
19346                 /* noprr=1 */
19347                 rack->rack_no_prr = 1;
19348                 /* lscwnd=0 */
19349                 rack->r_limit_scw = 0;
19350                 err = 0;
19351         } else if (prof == 0) {
19352                 /* This changes things back to the default settings */
19353                 err = 0;
19354                 if (rack->rc_always_pace) {
19355                         tcp_decrement_paced_conn();
19356                         rack_undo_cc_pacing(rack);
19357                         rack->rc_always_pace = 0;
19358                 }
19359                 if (rack_pace_every_seg && tcp_can_enable_pacing()) {
19360                         rack->rc_always_pace = 1;
19361                         if (rack->use_fixed_rate || rack->gp_ready)
19362                                 rack_set_cc_pacing(rack);
19363                 } else
19364                         rack->rc_always_pace = 0;
19365                 if (rack_dsack_std_based & 0x1) {
19366                         /* Basically this means all rack timers are at least (srtt + 1/4 srtt) */
19367                         rack->rc_rack_tmr_std_based = 1;
19368                 }
19369                 if (rack_dsack_std_based & 0x2) {
19370                         /* Basically this means  rack timers are extended based on dsack by up to (2 * srtt) */
19371                         rack->rc_rack_use_dsack = 1;
19372                 }
19373                 if (rack_use_cmp_acks)
19374                         rack->r_use_cmp_ack = 1;
19375                 else
19376                         rack->r_use_cmp_ack = 0;
19377                 if (rack_disable_prr)
19378                         rack->rack_no_prr = 1;
19379                 else
19380                         rack->rack_no_prr = 0;
19381                 if (rack_gp_no_rec_chg)
19382                         rack->rc_gp_no_rec_chg = 1;
19383                 else
19384                         rack->rc_gp_no_rec_chg = 0;
19385                 if (rack_enable_mqueue_for_nonpaced || rack->r_use_cmp_ack) {
19386                         rack->r_mbuf_queue = 1;
19387                         if (TCPS_HAVEESTABLISHED(rack->rc_tp->t_state))
19388                                 rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
19389                         rack->rc_inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
19390                 } else {
19391                         rack->r_mbuf_queue = 0;
19392                         rack->rc_inp->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
19393                 }
19394                 if (rack_enable_shared_cwnd)
19395                         rack->rack_enable_scwnd = 1;
19396                 else
19397                         rack->rack_enable_scwnd = 0;
19398                 if (rack_do_dyn_mul) {
19399                         /* When dynamic adjustment is on CA needs to start at 100% */
19400                         rack->rc_gp_dyn_mul = 1;
19401                         if (rack_do_dyn_mul >= 100)
19402                                 rack->r_ctl.rack_per_of_gp_ca = rack_do_dyn_mul;
19403                 } else {
19404                         rack->r_ctl.rack_per_of_gp_ca = rack_per_of_gp_ca;
19405                         rack->rc_gp_dyn_mul = 0;
19406                 }
19407                 rack->r_rr_config = 0;
19408                 rack->r_ctl.rc_no_push_at_mrtt = 0;
19409                 rack->rc_pace_to_cwnd = 0;
19410                 rack->rc_pace_fill_if_rttin_range = 0;
19411                 rack->rtt_limit_mul = 0;
19412
19413                 if (rack_enable_hw_pacing)
19414                         rack->rack_hdw_pace_ena = 1;
19415                 else
19416                         rack->rack_hdw_pace_ena = 0;
19417                 if (rack_disable_prr)
19418                         rack->rack_no_prr = 1;
19419                 else
19420                         rack->rack_no_prr = 0;
19421                 if (rack_limits_scwnd)
19422                         rack->r_limit_scw  = 1;
19423                 else
19424                         rack->r_limit_scw  = 0;
19425                 err = 0;
19426         }
19427         return (err);
19428 }
19429
19430 static int
19431 rack_add_deferred_option(struct tcp_rack *rack, int sopt_name, uint64_t loptval)
19432 {
19433         struct deferred_opt_list *dol;
19434
19435         dol = malloc(sizeof(struct deferred_opt_list),
19436                      M_TCPFSB, M_NOWAIT|M_ZERO);
19437         if (dol == NULL) {
19438                 /*
19439                  * No space yikes -- fail out..
19440                  */
19441                 return (0);
19442         }
19443         dol->optname = sopt_name;
19444         dol->optval = loptval;
19445         TAILQ_INSERT_TAIL(&rack->r_ctl.opt_list, dol, next);
19446         return (1);
19447 }
19448
19449 static int
19450 rack_process_option(struct tcpcb *tp, struct tcp_rack *rack, int sopt_name,
19451                     uint32_t optval, uint64_t loptval)
19452 {
19453         struct epoch_tracker et;
19454         struct sockopt sopt;
19455         struct cc_newreno_opts opt;
19456         uint64_t val;
19457         int error = 0;
19458         uint16_t ca, ss;
19459
19460         switch (sopt_name) {
19461
19462         case TCP_RACK_DSACK_OPT:
19463                 RACK_OPTS_INC(tcp_rack_dsack_opt);
19464                 if (optval & 0x1) {
19465                         rack->rc_rack_tmr_std_based = 1;
19466                 } else {
19467                         rack->rc_rack_tmr_std_based = 0;
19468                 }
19469                 if (optval & 0x2) {
19470                         rack->rc_rack_use_dsack = 1;
19471                 } else {
19472                         rack->rc_rack_use_dsack = 0;
19473                 }
19474                 rack_log_dsack_event(rack, 5, __LINE__, 0, 0);
19475                 break;
19476         case TCP_RACK_PACING_BETA:
19477                 RACK_OPTS_INC(tcp_rack_beta);
19478                 if (strcmp(tp->cc_algo->name, CCALGONAME_NEWRENO) != 0) {
19479                         /* This only works for newreno. */
19480                         error = EINVAL;
19481                         break;
19482                 }
19483                 if (rack->rc_pacing_cc_set) {
19484                         /*
19485                          * Set them into the real CC module
19486                          * whats in the rack pcb is the old values
19487                          * to be used on restoral/
19488                          */
19489                         sopt.sopt_dir = SOPT_SET;
19490                         opt.name = CC_NEWRENO_BETA;
19491                         opt.val = optval;
19492                         if (CC_ALGO(tp)->ctl_output != NULL)
19493                                 error = CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt);
19494                         else {
19495                                 error = ENOENT;
19496                                 break;
19497                         }
19498                 } else {
19499                         /*
19500                          * Not pacing yet so set it into our local
19501                          * rack pcb storage.
19502                          */
19503                         rack->r_ctl.rc_saved_beta.beta = optval;
19504                 }
19505                 break;
19506         case TCP_RACK_TIMER_SLOP:
19507                 RACK_OPTS_INC(tcp_rack_timer_slop);
19508                 rack->r_ctl.timer_slop = optval;
19509                 if (rack->rc_tp->t_srtt) {
19510                         /*
19511                          * If we have an SRTT lets update t_rxtcur
19512                          * to have the new slop.
19513                          */
19514                         RACK_TCPT_RANGESET(tp->t_rxtcur, RACK_REXMTVAL(tp),
19515                                            rack_rto_min, rack_rto_max,
19516                                            rack->r_ctl.timer_slop);
19517                 }
19518                 break;
19519         case TCP_RACK_PACING_BETA_ECN:
19520                 RACK_OPTS_INC(tcp_rack_beta_ecn);
19521                 if (strcmp(tp->cc_algo->name, CCALGONAME_NEWRENO) != 0) {
19522                         /* This only works for newreno. */
19523                         error = EINVAL;
19524                         break;
19525                 }
19526                 if (rack->rc_pacing_cc_set) {
19527                         /*
19528                          * Set them into the real CC module
19529                          * whats in the rack pcb is the old values
19530                          * to be used on restoral/
19531                          */
19532                         sopt.sopt_dir = SOPT_SET;
19533                         opt.name = CC_NEWRENO_BETA_ECN;
19534                         opt.val = optval;
19535                         if (CC_ALGO(tp)->ctl_output != NULL)
19536                                 error = CC_ALGO(tp)->ctl_output(tp->ccv, &sopt, &opt);
19537                         else
19538                                 error = ENOENT;
19539                 } else {
19540                         /*
19541                          * Not pacing yet so set it into our local
19542                          * rack pcb storage.
19543                          */
19544                         rack->r_ctl.rc_saved_beta.beta_ecn = optval;
19545                         rack->r_ctl.rc_saved_beta.newreno_flags = CC_NEWRENO_BETA_ECN_ENABLED;
19546                 }
19547                 break;
19548         case TCP_DEFER_OPTIONS:
19549                 RACK_OPTS_INC(tcp_defer_opt);
19550                 if (optval) {
19551                         if (rack->gp_ready) {
19552                                 /* Too late */
19553                                 error = EINVAL;
19554                                 break;
19555                         }
19556                         rack->defer_options = 1;
19557                 } else
19558                         rack->defer_options = 0;
19559                 break;
19560         case TCP_RACK_MEASURE_CNT:
19561                 RACK_OPTS_INC(tcp_rack_measure_cnt);
19562                 if (optval && (optval <= 0xff)) {
19563                         rack->r_ctl.req_measurements = optval;
19564                 } else
19565                         error = EINVAL;
19566                 break;
19567         case TCP_REC_ABC_VAL:
19568                 RACK_OPTS_INC(tcp_rec_abc_val);
19569                 if (optval > 0)
19570                         rack->r_use_labc_for_rec = 1;
19571                 else
19572                         rack->r_use_labc_for_rec = 0;
19573                 break;
19574         case TCP_RACK_ABC_VAL:
19575                 RACK_OPTS_INC(tcp_rack_abc_val);
19576                 if ((optval > 0) && (optval < 255))
19577                         rack->rc_labc = optval;
19578                 else
19579                         error = EINVAL;
19580                 break;
19581         case TCP_HDWR_UP_ONLY:
19582                 RACK_OPTS_INC(tcp_pacing_up_only);
19583                 if (optval)
19584                         rack->r_up_only = 1;
19585                 else
19586                         rack->r_up_only = 0;
19587                 break;
19588         case TCP_PACING_RATE_CAP:
19589                 RACK_OPTS_INC(tcp_pacing_rate_cap);
19590                 rack->r_ctl.bw_rate_cap = loptval;
19591                 break;
19592         case TCP_RACK_PROFILE:
19593                 RACK_OPTS_INC(tcp_profile);
19594                 error = rack_set_profile(rack, optval);
19595                 break;
19596         case TCP_USE_CMP_ACKS:
19597                 RACK_OPTS_INC(tcp_use_cmp_acks);
19598                 if ((optval == 0) && (rack->rc_inp->inp_flags2 & INP_MBUF_ACKCMP)) {
19599                         /* You can't turn it off once its on! */
19600                         error = EINVAL;
19601                 } else if ((optval == 1) && (rack->r_use_cmp_ack == 0)) {
19602                         rack->r_use_cmp_ack = 1;
19603                         rack->r_mbuf_queue = 1;
19604                         tp->t_inpcb->inp_flags2 |= INP_SUPPORTS_MBUFQ;
19605                 }
19606                 if (rack->r_use_cmp_ack && TCPS_HAVEESTABLISHED(tp->t_state))
19607                         rack->rc_inp->inp_flags2 |= INP_MBUF_ACKCMP;
19608                 break;
19609         case TCP_SHARED_CWND_TIME_LIMIT:
19610                 RACK_OPTS_INC(tcp_lscwnd);
19611                 if (optval)
19612                         rack->r_limit_scw = 1;
19613                 else
19614                         rack->r_limit_scw = 0;
19615                 break;
19616         case TCP_RACK_PACE_TO_FILL:
19617                 RACK_OPTS_INC(tcp_fillcw);
19618                 if (optval == 0)
19619                         rack->rc_pace_to_cwnd = 0;
19620                 else {
19621                         rack->rc_pace_to_cwnd = 1;
19622                         if (optval > 1)
19623                                 rack->r_fill_less_agg = 1;
19624                 }
19625                 if ((optval >= rack_gp_rtt_maxmul) &&
19626                     rack_gp_rtt_maxmul &&
19627                     (optval < 0xf)) {
19628                         rack->rc_pace_fill_if_rttin_range = 1;
19629                         rack->rtt_limit_mul = optval;
19630                 } else {
19631                         rack->rc_pace_fill_if_rttin_range = 0;
19632                         rack->rtt_limit_mul = 0;
19633                 }
19634                 break;
19635         case TCP_RACK_NO_PUSH_AT_MAX:
19636                 RACK_OPTS_INC(tcp_npush);
19637                 if (optval == 0)
19638                         rack->r_ctl.rc_no_push_at_mrtt = 0;
19639                 else if (optval < 0xff)
19640                         rack->r_ctl.rc_no_push_at_mrtt = optval;
19641                 else
19642                         error = EINVAL;
19643                 break;
19644         case TCP_SHARED_CWND_ENABLE:
19645                 RACK_OPTS_INC(tcp_rack_scwnd);
19646                 if (optval == 0)
19647                         rack->rack_enable_scwnd = 0;
19648                 else
19649                         rack->rack_enable_scwnd = 1;
19650                 break;
19651         case TCP_RACK_MBUF_QUEUE:
19652                 /* Now do we use the LRO mbuf-queue feature */
19653                 RACK_OPTS_INC(tcp_rack_mbufq);
19654                 if (optval || rack->r_use_cmp_ack)
19655                         rack->r_mbuf_queue = 1;
19656                 else
19657                         rack->r_mbuf_queue = 0;
19658                 if  (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
19659                         tp->t_inpcb->inp_flags2 |= INP_SUPPORTS_MBUFQ;
19660                 else
19661                         tp->t_inpcb->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
19662                 break;
19663         case TCP_RACK_NONRXT_CFG_RATE:
19664                 RACK_OPTS_INC(tcp_rack_cfg_rate);
19665                 if (optval == 0)
19666                         rack->rack_rec_nonrxt_use_cr = 0;
19667                 else
19668                         rack->rack_rec_nonrxt_use_cr = 1;
19669                 break;
19670         case TCP_NO_PRR:
19671                 RACK_OPTS_INC(tcp_rack_noprr);
19672                 if (optval == 0)
19673                         rack->rack_no_prr = 0;
19674                 else if (optval == 1)
19675                         rack->rack_no_prr = 1;
19676                 else if (optval == 2)
19677                         rack->no_prr_addback = 1;
19678                 else
19679                         error = EINVAL;
19680                 break;
19681         case TCP_TIMELY_DYN_ADJ:
19682                 RACK_OPTS_INC(tcp_timely_dyn);
19683                 if (optval == 0)
19684                         rack->rc_gp_dyn_mul = 0;
19685                 else {
19686                         rack->rc_gp_dyn_mul = 1;
19687                         if (optval >= 100) {
19688                                 /*
19689                                  * If the user sets something 100 or more
19690                                  * its the gp_ca value.
19691                                  */
19692                                 rack->r_ctl.rack_per_of_gp_ca  = optval;
19693                         }
19694                 }
19695                 break;
19696         case TCP_RACK_DO_DETECTION:
19697                 RACK_OPTS_INC(tcp_rack_do_detection);
19698                 if (optval == 0)
19699                         rack->do_detection = 0;
19700                 else
19701                         rack->do_detection = 1;
19702                 break;
19703         case TCP_RACK_TLP_USE:
19704                 if ((optval < TLP_USE_ID) || (optval > TLP_USE_TWO_TWO)) {
19705                         error = EINVAL;
19706                         break;
19707                 }
19708                 RACK_OPTS_INC(tcp_tlp_use);
19709                 rack->rack_tlp_threshold_use = optval;
19710                 break;
19711         case TCP_RACK_TLP_REDUCE:
19712                 /* RACK TLP cwnd reduction (bool) */
19713                 RACK_OPTS_INC(tcp_rack_tlp_reduce);
19714                 rack->r_ctl.rc_tlp_cwnd_reduce = optval;
19715                 break;
19716         /*  Pacing related ones */
19717         case TCP_RACK_PACE_ALWAYS:
19718                 /*
19719                  * zero is old rack method, 1 is new
19720                  * method using a pacing rate.
19721                  */
19722                 RACK_OPTS_INC(tcp_rack_pace_always);
19723                 if (optval > 0) {
19724                         if (rack->rc_always_pace) {
19725                                 error = EALREADY;
19726                                 break;
19727                         } else if (tcp_can_enable_pacing()) {
19728                                 rack->rc_always_pace = 1;
19729                                 if (rack->use_fixed_rate || rack->gp_ready)
19730                                         rack_set_cc_pacing(rack);
19731                         }
19732                         else {
19733                                 error = ENOSPC;
19734                                 break;
19735                         }
19736                 } else {
19737                         if (rack->rc_always_pace) {
19738                                 tcp_decrement_paced_conn();
19739                                 rack->rc_always_pace = 0;
19740                                 rack_undo_cc_pacing(rack);
19741                         }
19742                 }
19743                 if  (rack->r_mbuf_queue || rack->rc_always_pace || rack->r_use_cmp_ack)
19744                         tp->t_inpcb->inp_flags2 |= INP_SUPPORTS_MBUFQ;
19745                 else
19746                         tp->t_inpcb->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
19747                 /* A rate may be set irate or other, if so set seg size */
19748                 rack_update_seg(rack);
19749                 break;
19750         case TCP_BBR_RACK_INIT_RATE:
19751                 RACK_OPTS_INC(tcp_initial_rate);
19752                 val = optval;
19753                 /* Change from kbits per second to bytes per second */
19754                 val *= 1000;
19755                 val /= 8;
19756                 rack->r_ctl.init_rate = val;
19757                 if (rack->rc_init_win != rack_default_init_window) {
19758                         uint32_t win, snt;
19759
19760                         /*
19761                          * Options don't always get applied
19762                          * in the order you think. So in order
19763                          * to assure we update a cwnd we need
19764                          * to check and see if we are still
19765                          * where we should raise the cwnd.
19766                          */
19767                         win = rc_init_window(rack);
19768                         if (SEQ_GT(tp->snd_max, tp->iss))
19769                                 snt = tp->snd_max - tp->iss;
19770                         else
19771                                 snt = 0;
19772                         if ((snt < win) &&
19773                             (tp->snd_cwnd < win))
19774                                 tp->snd_cwnd = win;
19775                 }
19776                 if (rack->rc_always_pace)
19777                         rack_update_seg(rack);
19778                 break;
19779         case TCP_BBR_IWINTSO:
19780                 RACK_OPTS_INC(tcp_initial_win);
19781                 if (optval && (optval <= 0xff)) {
19782                         uint32_t win, snt;
19783
19784                         rack->rc_init_win = optval;
19785                         win = rc_init_window(rack);
19786                         if (SEQ_GT(tp->snd_max, tp->iss))
19787                                 snt = tp->snd_max - tp->iss;
19788                         else
19789                                 snt = 0;
19790                         if ((snt < win) &&
19791                             (tp->t_srtt |
19792 #ifdef NETFLIX_PEAKRATE
19793                              tp->t_maxpeakrate |
19794 #endif
19795                              rack->r_ctl.init_rate)) {
19796                                 /*
19797                                  * We are not past the initial window
19798                                  * and we have some bases for pacing,
19799                                  * so we need to possibly adjust up
19800                                  * the cwnd. Note even if we don't set
19801                                  * the cwnd, its still ok to raise the rc_init_win
19802                                  * which can be used coming out of idle when we
19803                                  * would have a rate.
19804                                  */
19805                                 if (tp->snd_cwnd < win)
19806                                         tp->snd_cwnd = win;
19807                         }
19808                         if (rack->rc_always_pace)
19809                                 rack_update_seg(rack);
19810                 } else
19811                         error = EINVAL;
19812                 break;
19813         case TCP_RACK_FORCE_MSEG:
19814                 RACK_OPTS_INC(tcp_rack_force_max_seg);
19815                 if (optval)
19816                         rack->rc_force_max_seg = 1;
19817                 else
19818                         rack->rc_force_max_seg = 0;
19819                 break;
19820         case TCP_RACK_PACE_MAX_SEG:
19821                 /* Max segments size in a pace in bytes */
19822                 RACK_OPTS_INC(tcp_rack_max_seg);
19823                 rack->rc_user_set_max_segs = optval;
19824                 rack_set_pace_segments(tp, rack, __LINE__, NULL);
19825                 break;
19826         case TCP_RACK_PACE_RATE_REC:
19827                 /* Set the fixed pacing rate in Bytes per second ca */
19828                 RACK_OPTS_INC(tcp_rack_pace_rate_rec);
19829                 rack->r_ctl.rc_fixed_pacing_rate_rec = optval;
19830                 if (rack->r_ctl.rc_fixed_pacing_rate_ca == 0)
19831                         rack->r_ctl.rc_fixed_pacing_rate_ca = optval;
19832                 if (rack->r_ctl.rc_fixed_pacing_rate_ss == 0)
19833                         rack->r_ctl.rc_fixed_pacing_rate_ss = optval;
19834                 rack->use_fixed_rate = 1;
19835                 if (rack->rc_always_pace)
19836                         rack_set_cc_pacing(rack);
19837                 rack_log_pacing_delay_calc(rack,
19838                                            rack->r_ctl.rc_fixed_pacing_rate_ss,
19839                                            rack->r_ctl.rc_fixed_pacing_rate_ca,
19840                                            rack->r_ctl.rc_fixed_pacing_rate_rec, 0, 0, 8,
19841                                            __LINE__, NULL,0);
19842                 break;
19843
19844         case TCP_RACK_PACE_RATE_SS:
19845                 /* Set the fixed pacing rate in Bytes per second ca */
19846                 RACK_OPTS_INC(tcp_rack_pace_rate_ss);
19847                 rack->r_ctl.rc_fixed_pacing_rate_ss = optval;
19848                 if (rack->r_ctl.rc_fixed_pacing_rate_ca == 0)
19849                         rack->r_ctl.rc_fixed_pacing_rate_ca = optval;
19850                 if (rack->r_ctl.rc_fixed_pacing_rate_rec == 0)
19851                         rack->r_ctl.rc_fixed_pacing_rate_rec = optval;
19852                 rack->use_fixed_rate = 1;
19853                 if (rack->rc_always_pace)
19854                         rack_set_cc_pacing(rack);
19855                 rack_log_pacing_delay_calc(rack,
19856                                            rack->r_ctl.rc_fixed_pacing_rate_ss,
19857                                            rack->r_ctl.rc_fixed_pacing_rate_ca,
19858                                            rack->r_ctl.rc_fixed_pacing_rate_rec, 0, 0, 8,
19859                                            __LINE__, NULL, 0);
19860                 break;
19861
19862         case TCP_RACK_PACE_RATE_CA:
19863                 /* Set the fixed pacing rate in Bytes per second ca */
19864                 RACK_OPTS_INC(tcp_rack_pace_rate_ca);
19865                 rack->r_ctl.rc_fixed_pacing_rate_ca = optval;
19866                 if (rack->r_ctl.rc_fixed_pacing_rate_ss == 0)
19867                         rack->r_ctl.rc_fixed_pacing_rate_ss = optval;
19868                 if (rack->r_ctl.rc_fixed_pacing_rate_rec == 0)
19869                         rack->r_ctl.rc_fixed_pacing_rate_rec = optval;
19870                 rack->use_fixed_rate = 1;
19871                 if (rack->rc_always_pace)
19872                         rack_set_cc_pacing(rack);
19873                 rack_log_pacing_delay_calc(rack,
19874                                            rack->r_ctl.rc_fixed_pacing_rate_ss,
19875                                            rack->r_ctl.rc_fixed_pacing_rate_ca,
19876                                            rack->r_ctl.rc_fixed_pacing_rate_rec, 0, 0, 8,
19877                                            __LINE__, NULL, 0);
19878                 break;
19879         case TCP_RACK_GP_INCREASE_REC:
19880                 RACK_OPTS_INC(tcp_gp_inc_rec);
19881                 rack->r_ctl.rack_per_of_gp_rec = optval;
19882                 rack_log_pacing_delay_calc(rack,
19883                                            rack->r_ctl.rack_per_of_gp_ss,
19884                                            rack->r_ctl.rack_per_of_gp_ca,
19885                                            rack->r_ctl.rack_per_of_gp_rec, 0, 0, 1,
19886                                            __LINE__, NULL, 0);
19887                 break;
19888         case TCP_RACK_GP_INCREASE_CA:
19889                 RACK_OPTS_INC(tcp_gp_inc_ca);
19890                 ca = optval;
19891                 if (ca < 100) {
19892                         /*
19893                          * We don't allow any reduction
19894                          * over the GP b/w.
19895                          */
19896                         error = EINVAL;
19897                         break;
19898                 }
19899                 rack->r_ctl.rack_per_of_gp_ca = ca;
19900                 rack_log_pacing_delay_calc(rack,
19901                                            rack->r_ctl.rack_per_of_gp_ss,
19902                                            rack->r_ctl.rack_per_of_gp_ca,
19903                                            rack->r_ctl.rack_per_of_gp_rec, 0, 0, 1,
19904                                            __LINE__, NULL, 0);
19905                 break;
19906         case TCP_RACK_GP_INCREASE_SS:
19907                 RACK_OPTS_INC(tcp_gp_inc_ss);
19908                 ss = optval;
19909                 if (ss < 100) {
19910                         /*
19911                          * We don't allow any reduction
19912                          * over the GP b/w.
19913                          */
19914                         error = EINVAL;
19915                         break;
19916                 }
19917                 rack->r_ctl.rack_per_of_gp_ss = ss;
19918                 rack_log_pacing_delay_calc(rack,
19919                                            rack->r_ctl.rack_per_of_gp_ss,
19920                                            rack->r_ctl.rack_per_of_gp_ca,
19921                                            rack->r_ctl.rack_per_of_gp_rec, 0, 0, 1,
19922                                            __LINE__, NULL, 0);
19923                 break;
19924         case TCP_RACK_RR_CONF:
19925                 RACK_OPTS_INC(tcp_rack_rrr_no_conf_rate);
19926                 if (optval && optval <= 3)
19927                         rack->r_rr_config = optval;
19928                 else
19929                         rack->r_rr_config = 0;
19930                 break;
19931         case TCP_HDWR_RATE_CAP:
19932                 RACK_OPTS_INC(tcp_hdwr_rate_cap);
19933                 if (optval) {
19934                         if (rack->r_rack_hw_rate_caps == 0)
19935                                 rack->r_rack_hw_rate_caps = 1;
19936                         else
19937                                 error = EALREADY;
19938                 } else {
19939                         rack->r_rack_hw_rate_caps = 0;
19940                 }
19941                 break;
19942         case TCP_BBR_HDWR_PACE:
19943                 RACK_OPTS_INC(tcp_hdwr_pacing);
19944                 if (optval){
19945                         if (rack->rack_hdrw_pacing == 0) {
19946                                 rack->rack_hdw_pace_ena = 1;
19947                                 rack->rack_attempt_hdwr_pace = 0;
19948                         } else
19949                                 error = EALREADY;
19950                 } else {
19951                         rack->rack_hdw_pace_ena = 0;
19952 #ifdef RATELIMIT
19953                         if (rack->r_ctl.crte != NULL) {
19954                                 rack->rack_hdrw_pacing = 0;
19955                                 rack->rack_attempt_hdwr_pace = 0;
19956                                 tcp_rel_pacing_rate(rack->r_ctl.crte, tp);
19957                                 rack->r_ctl.crte = NULL;
19958                         }
19959 #endif
19960                 }
19961                 break;
19962         /*  End Pacing related ones */
19963         case TCP_RACK_PRR_SENDALOT:
19964                 /* Allow PRR to send more than one seg */
19965                 RACK_OPTS_INC(tcp_rack_prr_sendalot);
19966                 rack->r_ctl.rc_prr_sendalot = optval;
19967                 break;
19968         case TCP_RACK_MIN_TO:
19969                 /* Minimum time between rack t-o's in ms */
19970                 RACK_OPTS_INC(tcp_rack_min_to);
19971                 rack->r_ctl.rc_min_to = optval;
19972                 break;
19973         case TCP_RACK_EARLY_SEG:
19974                 /* If early recovery max segments */
19975                 RACK_OPTS_INC(tcp_rack_early_seg);
19976                 rack->r_ctl.rc_early_recovery_segs = optval;
19977                 break;
19978         case TCP_RACK_ENABLE_HYSTART:
19979         {
19980                 if (optval) {
19981                         tp->ccv->flags |= CCF_HYSTART_ALLOWED;
19982                         if (rack_do_hystart > RACK_HYSTART_ON)
19983                                 tp->ccv->flags |= CCF_HYSTART_CAN_SH_CWND;
19984                         if (rack_do_hystart > RACK_HYSTART_ON_W_SC)
19985                                 tp->ccv->flags |= CCF_HYSTART_CONS_SSTH;
19986                 } else {
19987                         tp->ccv->flags &= ~(CCF_HYSTART_ALLOWED|CCF_HYSTART_CAN_SH_CWND|CCF_HYSTART_CONS_SSTH);
19988                 }
19989         }
19990         break;
19991         case TCP_RACK_REORD_THRESH:
19992                 /* RACK reorder threshold (shift amount) */
19993                 RACK_OPTS_INC(tcp_rack_reord_thresh);
19994                 if ((optval > 0) && (optval < 31))
19995                         rack->r_ctl.rc_reorder_shift = optval;
19996                 else
19997                         error = EINVAL;
19998                 break;
19999         case TCP_RACK_REORD_FADE:
20000                 /* Does reordering fade after ms time */
20001                 RACK_OPTS_INC(tcp_rack_reord_fade);
20002                 rack->r_ctl.rc_reorder_fade = optval;
20003                 break;
20004         case TCP_RACK_TLP_THRESH:
20005                 /* RACK TLP theshold i.e. srtt+(srtt/N) */
20006                 RACK_OPTS_INC(tcp_rack_tlp_thresh);
20007                 if (optval)
20008                         rack->r_ctl.rc_tlp_threshold = optval;
20009                 else
20010                         error = EINVAL;
20011                 break;
20012         case TCP_BBR_USE_RACK_RR:
20013                 RACK_OPTS_INC(tcp_rack_rr);
20014                 if (optval)
20015                         rack->use_rack_rr = 1;
20016                 else
20017                         rack->use_rack_rr = 0;
20018                 break;
20019         case TCP_FAST_RSM_HACK:
20020                 RACK_OPTS_INC(tcp_rack_fastrsm_hack);
20021                 if (optval)
20022                         rack->fast_rsm_hack = 1;
20023                 else
20024                         rack->fast_rsm_hack = 0;
20025                 break;
20026         case TCP_RACK_PKT_DELAY:
20027                 /* RACK added ms i.e. rack-rtt + reord + N */
20028                 RACK_OPTS_INC(tcp_rack_pkt_delay);
20029                 rack->r_ctl.rc_pkt_delay = optval;
20030                 break;
20031         case TCP_DELACK:
20032                 RACK_OPTS_INC(tcp_rack_delayed_ack);
20033                 if (optval == 0)
20034                         tp->t_delayed_ack = 0;
20035                 else
20036                         tp->t_delayed_ack = 1;
20037                 if (tp->t_flags & TF_DELACK) {
20038                         tp->t_flags &= ~TF_DELACK;
20039                         tp->t_flags |= TF_ACKNOW;
20040                         NET_EPOCH_ENTER(et);
20041                         rack_output(tp);
20042                         NET_EPOCH_EXIT(et);
20043                 }
20044                 break;
20045
20046         case TCP_BBR_RACK_RTT_USE:
20047                 RACK_OPTS_INC(tcp_rack_rtt_use);
20048                 if ((optval != USE_RTT_HIGH) &&
20049                     (optval != USE_RTT_LOW) &&
20050                     (optval != USE_RTT_AVG))
20051                         error = EINVAL;
20052                 else
20053                         rack->r_ctl.rc_rate_sample_method = optval;
20054                 break;
20055         case TCP_DATA_AFTER_CLOSE:
20056                 RACK_OPTS_INC(tcp_data_after_close);
20057                 if (optval)
20058                         rack->rc_allow_data_af_clo = 1;
20059                 else
20060                         rack->rc_allow_data_af_clo = 0;
20061                 break;
20062         default:
20063                 break;
20064         }
20065 #ifdef NETFLIX_STATS
20066         tcp_log_socket_option(tp, sopt_name, optval, error);
20067 #endif
20068         return (error);
20069 }
20070
20071
20072 static void
20073 rack_apply_deferred_options(struct tcp_rack *rack)
20074 {
20075         struct deferred_opt_list *dol, *sdol;
20076         uint32_t s_optval;
20077
20078         TAILQ_FOREACH_SAFE(dol, &rack->r_ctl.opt_list, next, sdol) {
20079                 TAILQ_REMOVE(&rack->r_ctl.opt_list, dol, next);
20080                 /* Disadvantage of deferal is you loose the error return */
20081                 s_optval = (uint32_t)dol->optval;
20082                 (void)rack_process_option(rack->rc_tp, rack, dol->optname, s_optval, dol->optval);
20083                 free(dol, M_TCPDO);
20084         }
20085 }
20086
20087 static void
20088 rack_hw_tls_change(struct tcpcb *tp, int chg)
20089 {
20090         /*
20091          * HW tls state has changed.. fix all
20092          * rsm's in flight.
20093          */
20094         struct tcp_rack *rack;
20095         struct rack_sendmap *rsm;
20096
20097         rack = (struct tcp_rack *)tp->t_fb_ptr;
20098         RB_FOREACH(rsm, rack_rb_tree_head, &rack->r_ctl.rc_mtree) {
20099                 if (chg)
20100                         rsm->r_hw_tls = 1;
20101                 else
20102                         rsm->r_hw_tls = 0;
20103         }
20104         if (chg)
20105                 rack->r_ctl.fsb.hw_tls = 1;
20106         else
20107                 rack->r_ctl.fsb.hw_tls = 0;
20108 }
20109
20110 static int
20111 rack_pru_options(struct tcpcb *tp, int flags)
20112 {
20113         if (flags & PRUS_OOB)
20114                 return (EOPNOTSUPP);
20115         return (0);
20116 }
20117
20118 static struct tcp_function_block __tcp_rack = {
20119         .tfb_tcp_block_name = __XSTRING(STACKNAME),
20120         .tfb_tcp_output = rack_output,
20121         .tfb_do_queued_segments = ctf_do_queued_segments,
20122         .tfb_do_segment_nounlock = rack_do_segment_nounlock,
20123         .tfb_tcp_do_segment = rack_do_segment,
20124         .tfb_tcp_ctloutput = rack_ctloutput,
20125         .tfb_tcp_fb_init = rack_init,
20126         .tfb_tcp_fb_fini = rack_fini,
20127         .tfb_tcp_timer_stop_all = rack_stopall,
20128         .tfb_tcp_timer_activate = rack_timer_activate,
20129         .tfb_tcp_timer_active = rack_timer_active,
20130         .tfb_tcp_timer_stop = rack_timer_stop,
20131         .tfb_tcp_rexmit_tmr = rack_remxt_tmr,
20132         .tfb_tcp_handoff_ok = rack_handoff_ok,
20133         .tfb_tcp_mtu_chg = rack_mtu_change,
20134         .tfb_pru_options = rack_pru_options,
20135         .tfb_hwtls_change = rack_hw_tls_change,
20136         .tfb_flags = TCP_FUNC_OUTPUT_CANDROP,
20137 };
20138
20139 /*
20140  * rack_ctloutput() must drop the inpcb lock before performing copyin on
20141  * socket option arguments.  When it re-acquires the lock after the copy, it
20142  * has to revalidate that the connection is still valid for the socket
20143  * option.
20144  */
20145 static int
20146 rack_set_sockopt(struct inpcb *inp, struct sockopt *sopt)
20147 {
20148 #ifdef INET6
20149         struct ip6_hdr *ip6;
20150 #endif
20151 #ifdef INET
20152         struct ip *ip;
20153 #endif
20154         struct tcpcb *tp;
20155         struct tcp_rack *rack;
20156         uint64_t loptval;
20157         int32_t error = 0, optval;
20158
20159         tp = intotcpcb(inp);
20160         rack = (struct tcp_rack *)tp->t_fb_ptr;
20161         if (rack == NULL) {
20162                 INP_WUNLOCK(inp);
20163                 return (EINVAL);
20164         }
20165 #ifdef INET6
20166         ip6 = (struct ip6_hdr *)rack->r_ctl.fsb.tcp_ip_hdr;
20167 #endif
20168 #ifdef INET
20169         ip = (struct ip *)rack->r_ctl.fsb.tcp_ip_hdr;
20170 #endif
20171
20172         switch (sopt->sopt_level) {
20173 #ifdef INET6
20174         case IPPROTO_IPV6:
20175                 MPASS(inp->inp_vflag & INP_IPV6PROTO);
20176                 switch (sopt->sopt_name) {
20177                 case IPV6_USE_MIN_MTU:
20178                         tcp6_use_min_mtu(tp);
20179                         break;
20180                 case IPV6_TCLASS:
20181                         /*
20182                          * The DSCP codepoint has changed, update the fsb.
20183                          */
20184                         ip6->ip6_flow = (ip6->ip6_flow & ~IPV6_FLOWINFO_MASK) |
20185                             (rack->rc_inp->inp_flow & IPV6_FLOWINFO_MASK);
20186                         break;
20187                 }
20188                 INP_WUNLOCK(inp);
20189                 return (0);
20190 #endif
20191 #ifdef INET
20192         case IPPROTO_IP:
20193                 switch (sopt->sopt_name) {
20194                 case IP_TOS:
20195                         /*
20196                          * The DSCP codepoint has changed, update the fsb.
20197                          */
20198                         ip->ip_tos = rack->rc_inp->inp_ip_tos;
20199                         break;
20200                 case IP_TTL:
20201                         /*
20202                          * The TTL has changed, update the fsb.
20203                          */
20204                         ip->ip_ttl = rack->rc_inp->inp_ip_ttl;
20205                         break;
20206                 }
20207                 INP_WUNLOCK(inp);
20208                 return (0);
20209 #endif
20210         }
20211
20212         switch (sopt->sopt_name) {
20213         case TCP_RACK_TLP_REDUCE:               /*  URL:tlp_reduce */
20214         /*  Pacing related ones */
20215         case TCP_RACK_PACE_ALWAYS:              /*  URL:pace_always */
20216         case TCP_BBR_RACK_INIT_RATE:            /*  URL:irate */
20217         case TCP_BBR_IWINTSO:                   /*  URL:tso_iwin */
20218         case TCP_RACK_PACE_MAX_SEG:             /*  URL:pace_max_seg */
20219         case TCP_RACK_FORCE_MSEG:               /*  URL:force_max_seg */
20220         case TCP_RACK_PACE_RATE_CA:             /*  URL:pr_ca */
20221         case TCP_RACK_PACE_RATE_SS:             /*  URL:pr_ss*/
20222         case TCP_RACK_PACE_RATE_REC:            /*  URL:pr_rec */
20223         case TCP_RACK_GP_INCREASE_CA:           /*  URL:gp_inc_ca */
20224         case TCP_RACK_GP_INCREASE_SS:           /*  URL:gp_inc_ss */
20225         case TCP_RACK_GP_INCREASE_REC:          /*  URL:gp_inc_rec */
20226         case TCP_RACK_RR_CONF:                  /*  URL:rrr_conf */
20227         case TCP_BBR_HDWR_PACE:                 /*  URL:hdwrpace */
20228         case TCP_HDWR_RATE_CAP:                 /*  URL:hdwrcap boolean */
20229         case TCP_PACING_RATE_CAP:               /*  URL:cap  -- used by side-channel */
20230         case TCP_HDWR_UP_ONLY:                  /*  URL:uponly -- hardware pacing  boolean */
20231        /* End pacing related */
20232         case TCP_FAST_RSM_HACK:                 /*  URL:frsm_hack */
20233         case TCP_DELACK:                        /*  URL:delack (in base TCP i.e. tcp_hints along with cc etc ) */
20234         case TCP_RACK_PRR_SENDALOT:             /*  URL:prr_sendalot */
20235         case TCP_RACK_MIN_TO:                   /*  URL:min_to */
20236         case TCP_RACK_EARLY_SEG:                /*  URL:early_seg */
20237         case TCP_RACK_REORD_THRESH:             /*  URL:reord_thresh */
20238         case TCP_RACK_REORD_FADE:               /*  URL:reord_fade */
20239         case TCP_RACK_TLP_THRESH:               /*  URL:tlp_thresh */
20240         case TCP_RACK_PKT_DELAY:                /*  URL:pkt_delay */
20241         case TCP_RACK_TLP_USE:                  /*  URL:tlp_use */
20242         case TCP_BBR_RACK_RTT_USE:              /*  URL:rttuse */
20243         case TCP_BBR_USE_RACK_RR:               /*  URL:rackrr */
20244         case TCP_RACK_DO_DETECTION:             /*  URL:detect */
20245         case TCP_NO_PRR:                        /*  URL:noprr */
20246         case TCP_TIMELY_DYN_ADJ:                /*  URL:dynamic */
20247         case TCP_DATA_AFTER_CLOSE:              /*  no URL */
20248         case TCP_RACK_NONRXT_CFG_RATE:          /*  URL:nonrxtcr */
20249         case TCP_SHARED_CWND_ENABLE:            /*  URL:scwnd */
20250         case TCP_RACK_MBUF_QUEUE:               /*  URL:mqueue */
20251         case TCP_RACK_NO_PUSH_AT_MAX:           /*  URL:npush */
20252         case TCP_RACK_PACE_TO_FILL:             /*  URL:fillcw */
20253         case TCP_SHARED_CWND_TIME_LIMIT:        /*  URL:lscwnd */
20254         case TCP_RACK_PROFILE:                  /*  URL:profile */
20255         case TCP_USE_CMP_ACKS:                  /*  URL:cmpack */
20256         case TCP_RACK_ABC_VAL:                  /*  URL:labc */
20257         case TCP_REC_ABC_VAL:                   /*  URL:reclabc */
20258         case TCP_RACK_MEASURE_CNT:              /*  URL:measurecnt */
20259         case TCP_DEFER_OPTIONS:                 /*  URL:defer */
20260         case TCP_RACK_DSACK_OPT:                /*  URL:dsack */
20261         case TCP_RACK_PACING_BETA:              /*  URL:pacing_beta */
20262         case TCP_RACK_PACING_BETA_ECN:          /*  URL:pacing_beta_ecn */
20263         case TCP_RACK_TIMER_SLOP:               /*  URL:timer_slop */
20264         case TCP_RACK_ENABLE_HYSTART:           /*  URL:hystart */
20265                 break;
20266         default:
20267                 /* Filter off all unknown options to the base stack */
20268                 return (tcp_default_ctloutput(inp, sopt));
20269                 break;
20270         }
20271         INP_WUNLOCK(inp);
20272         if (sopt->sopt_name == TCP_PACING_RATE_CAP) {
20273                 error = sooptcopyin(sopt, &loptval, sizeof(loptval), sizeof(loptval));
20274                 /*
20275                  * We truncate it down to 32 bits for the socket-option trace this
20276                  * means rates > 34Gbps won't show right, but thats probably ok.
20277                  */
20278                 optval = (uint32_t)loptval;
20279         } else {
20280                 error = sooptcopyin(sopt, &optval, sizeof(optval), sizeof(optval));
20281                 /* Save it in 64 bit form too */
20282                 loptval = optval;
20283         }
20284         if (error)
20285                 return (error);
20286         INP_WLOCK(inp);
20287         if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
20288                 INP_WUNLOCK(inp);
20289                 return (ECONNRESET);
20290         }
20291         if (tp->t_fb != &__tcp_rack) {
20292                 INP_WUNLOCK(inp);
20293                 return (ENOPROTOOPT);
20294         }
20295         if (rack->defer_options && (rack->gp_ready == 0) &&
20296             (sopt->sopt_name != TCP_DEFER_OPTIONS) &&
20297             (sopt->sopt_name != TCP_RACK_PACING_BETA) &&
20298             (sopt->sopt_name != TCP_RACK_PACING_BETA_ECN) &&
20299             (sopt->sopt_name != TCP_RACK_MEASURE_CNT)) {
20300                 /* Options are beind deferred */
20301                 if (rack_add_deferred_option(rack, sopt->sopt_name, loptval)) {
20302                         INP_WUNLOCK(inp);
20303                         return (0);
20304                 } else {
20305                         /* No memory to defer, fail */
20306                         INP_WUNLOCK(inp);
20307                         return (ENOMEM);
20308                 }
20309         }
20310         error = rack_process_option(tp, rack, sopt->sopt_name, optval, loptval);
20311         INP_WUNLOCK(inp);
20312         return (error);
20313 }
20314
20315 static void
20316 rack_fill_info(struct tcpcb *tp, struct tcp_info *ti)
20317 {
20318
20319         INP_WLOCK_ASSERT(tp->t_inpcb);
20320         bzero(ti, sizeof(*ti));
20321
20322         ti->tcpi_state = tp->t_state;
20323         if ((tp->t_flags & TF_REQ_TSTMP) && (tp->t_flags & TF_RCVD_TSTMP))
20324                 ti->tcpi_options |= TCPI_OPT_TIMESTAMPS;
20325         if (tp->t_flags & TF_SACK_PERMIT)
20326                 ti->tcpi_options |= TCPI_OPT_SACK;
20327         if ((tp->t_flags & TF_REQ_SCALE) && (tp->t_flags & TF_RCVD_SCALE)) {
20328                 ti->tcpi_options |= TCPI_OPT_WSCALE;
20329                 ti->tcpi_snd_wscale = tp->snd_scale;
20330                 ti->tcpi_rcv_wscale = tp->rcv_scale;
20331         }
20332         if (tp->t_flags2 & TF2_ECN_PERMIT)
20333                 ti->tcpi_options |= TCPI_OPT_ECN;
20334         if (tp->t_flags & TF_FASTOPEN)
20335                 ti->tcpi_options |= TCPI_OPT_TFO;
20336         /* still kept in ticks is t_rcvtime */
20337         ti->tcpi_last_data_recv = ((uint32_t)ticks - tp->t_rcvtime) * tick;
20338         /* Since we hold everything in precise useconds this is easy */
20339         ti->tcpi_rtt = tp->t_srtt;
20340         ti->tcpi_rttvar = tp->t_rttvar;
20341         ti->tcpi_rto = tp->t_rxtcur;
20342         ti->tcpi_snd_ssthresh = tp->snd_ssthresh;
20343         ti->tcpi_snd_cwnd = tp->snd_cwnd;
20344         /*
20345          * FreeBSD-specific extension fields for tcp_info.
20346          */
20347         ti->tcpi_rcv_space = tp->rcv_wnd;
20348         ti->tcpi_rcv_nxt = tp->rcv_nxt;
20349         ti->tcpi_snd_wnd = tp->snd_wnd;
20350         ti->tcpi_snd_bwnd = 0;          /* Unused, kept for compat. */
20351         ti->tcpi_snd_nxt = tp->snd_nxt;
20352         ti->tcpi_snd_mss = tp->t_maxseg;
20353         ti->tcpi_rcv_mss = tp->t_maxseg;
20354         ti->tcpi_snd_rexmitpack = tp->t_sndrexmitpack;
20355         ti->tcpi_rcv_ooopack = tp->t_rcvoopack;
20356         ti->tcpi_snd_zerowin = tp->t_sndzerowin;
20357 #ifdef NETFLIX_STATS
20358         ti->tcpi_total_tlp = tp->t_sndtlppack;
20359         ti->tcpi_total_tlp_bytes = tp->t_sndtlpbyte;
20360         memcpy(&ti->tcpi_rxsyninfo, &tp->t_rxsyninfo, sizeof(struct tcpsyninfo));
20361 #endif
20362 #ifdef TCP_OFFLOAD
20363         if (tp->t_flags & TF_TOE) {
20364                 ti->tcpi_options |= TCPI_OPT_TOE;
20365                 tcp_offload_tcp_info(tp, ti);
20366         }
20367 #endif
20368 }
20369
20370 static int
20371 rack_get_sockopt(struct inpcb *inp, struct sockopt *sopt)
20372 {
20373         struct tcpcb *tp;
20374         struct tcp_rack *rack;
20375         int32_t error, optval;
20376         uint64_t val, loptval;
20377         struct  tcp_info ti;
20378         /*
20379          * Because all our options are either boolean or an int, we can just
20380          * pull everything into optval and then unlock and copy. If we ever
20381          * add a option that is not a int, then this will have quite an
20382          * impact to this routine.
20383          */
20384         error = 0;
20385         tp = intotcpcb(inp);
20386         rack = (struct tcp_rack *)tp->t_fb_ptr;
20387         if (rack == NULL) {
20388                 INP_WUNLOCK(inp);
20389                 return (EINVAL);
20390         }
20391         switch (sopt->sopt_name) {
20392         case TCP_INFO:
20393                 /* First get the info filled */
20394                 rack_fill_info(tp, &ti);
20395                 /* Fix up the rtt related fields if needed */
20396                 INP_WUNLOCK(inp);
20397                 error = sooptcopyout(sopt, &ti, sizeof ti);
20398                 return (error);
20399         /*
20400          * Beta is the congestion control value for NewReno that influences how
20401          * much of a backoff happens when loss is detected. It is normally set
20402          * to 50 for 50% i.e. the cwnd is reduced to 50% of its previous value
20403          * when you exit recovery.
20404          */
20405         case TCP_RACK_PACING_BETA:
20406                 if (strcmp(tp->cc_algo->name, CCALGONAME_NEWRENO) != 0)
20407                         error = EINVAL;
20408                 else if (rack->rc_pacing_cc_set == 0)
20409                         optval = rack->r_ctl.rc_saved_beta.beta;
20410                 else {
20411                         /*
20412                          * Reach out into the CC data and report back what
20413                          * I have previously set. Yeah it looks hackish but
20414                          * we don't want to report the saved values.
20415                          */
20416                         if (tp->ccv->cc_data)
20417                                 optval = ((struct newreno *)tp->ccv->cc_data)->beta;
20418                         else
20419                                 error = EINVAL;
20420                 }
20421                 break;
20422                 /*
20423                  * Beta_ecn is the congestion control value for NewReno that influences how
20424                  * much of a backoff happens when a ECN mark is detected. It is normally set
20425                  * to 80 for 80% i.e. the cwnd is reduced by 20% of its previous value when
20426                  * you exit recovery. Note that classic ECN has a beta of 50, it is only
20427                  * ABE Ecn that uses this "less" value, but we do too with pacing :)
20428                  */
20429
20430         case TCP_RACK_PACING_BETA_ECN:
20431                 if (strcmp(tp->cc_algo->name, CCALGONAME_NEWRENO) != 0)
20432                         error = EINVAL;
20433                 else if (rack->rc_pacing_cc_set == 0)
20434                         optval = rack->r_ctl.rc_saved_beta.beta_ecn;
20435                 else {
20436                         /*
20437                          * Reach out into the CC data and report back what
20438                          * I have previously set. Yeah it looks hackish but
20439                          * we don't want to report the saved values.
20440                          */
20441                         if (tp->ccv->cc_data)
20442                                 optval = ((struct newreno *)tp->ccv->cc_data)->beta_ecn;
20443                         else
20444                                 error = EINVAL;
20445                 }
20446                 break;
20447         case TCP_RACK_DSACK_OPT:
20448                 optval = 0;
20449                 if (rack->rc_rack_tmr_std_based) {
20450                         optval |= 1;
20451                 }
20452                 if (rack->rc_rack_use_dsack) {
20453                         optval |= 2;
20454                 }
20455                 break;
20456         case TCP_RACK_ENABLE_HYSTART:
20457         {
20458                 if (tp->ccv->flags & CCF_HYSTART_ALLOWED) {
20459                         optval = RACK_HYSTART_ON;
20460                         if (tp->ccv->flags & CCF_HYSTART_CAN_SH_CWND)
20461                                 optval = RACK_HYSTART_ON_W_SC;
20462                         if (tp->ccv->flags & CCF_HYSTART_CONS_SSTH)
20463                                 optval = RACK_HYSTART_ON_W_SC_C;
20464                 } else {
20465                         optval = RACK_HYSTART_OFF;
20466                 }
20467         }
20468         break;
20469         case TCP_FAST_RSM_HACK:
20470                 optval = rack->fast_rsm_hack;
20471                 break;
20472         case TCP_DEFER_OPTIONS:
20473                 optval = rack->defer_options;
20474                 break;
20475         case TCP_RACK_MEASURE_CNT:
20476                 optval = rack->r_ctl.req_measurements;
20477                 break;
20478         case TCP_REC_ABC_VAL:
20479                 optval = rack->r_use_labc_for_rec;
20480                 break;
20481         case TCP_RACK_ABC_VAL:
20482                 optval = rack->rc_labc;
20483                 break;
20484         case TCP_HDWR_UP_ONLY:
20485                 optval= rack->r_up_only;
20486                 break;
20487         case TCP_PACING_RATE_CAP:
20488                 loptval = rack->r_ctl.bw_rate_cap;
20489                 break;
20490         case TCP_RACK_PROFILE:
20491                 /* You cannot retrieve a profile, its write only */
20492                 error = EINVAL;
20493                 break;
20494         case TCP_USE_CMP_ACKS:
20495                 optval = rack->r_use_cmp_ack;
20496                 break;
20497         case TCP_RACK_PACE_TO_FILL:
20498                 optval = rack->rc_pace_to_cwnd;
20499                 if (optval && rack->r_fill_less_agg)
20500                         optval++;
20501                 break;
20502         case TCP_RACK_NO_PUSH_AT_MAX:
20503                 optval = rack->r_ctl.rc_no_push_at_mrtt;
20504                 break;
20505         case TCP_SHARED_CWND_ENABLE:
20506                 optval = rack->rack_enable_scwnd;
20507                 break;
20508         case TCP_RACK_NONRXT_CFG_RATE:
20509                 optval = rack->rack_rec_nonrxt_use_cr;
20510                 break;
20511         case TCP_NO_PRR:
20512                 if (rack->rack_no_prr  == 1)
20513                         optval = 1;
20514                 else if (rack->no_prr_addback == 1)
20515                         optval = 2;
20516                 else
20517                         optval = 0;
20518                 break;
20519         case TCP_RACK_DO_DETECTION:
20520                 optval = rack->do_detection;
20521                 break;
20522         case TCP_RACK_MBUF_QUEUE:
20523                 /* Now do we use the LRO mbuf-queue feature */
20524                 optval = rack->r_mbuf_queue;
20525                 break;
20526         case TCP_TIMELY_DYN_ADJ:
20527                 optval = rack->rc_gp_dyn_mul;
20528                 break;
20529         case TCP_BBR_IWINTSO:
20530                 optval = rack->rc_init_win;
20531                 break;
20532         case TCP_RACK_TLP_REDUCE:
20533                 /* RACK TLP cwnd reduction (bool) */
20534                 optval = rack->r_ctl.rc_tlp_cwnd_reduce;
20535                 break;
20536         case TCP_BBR_RACK_INIT_RATE:
20537                 val = rack->r_ctl.init_rate;
20538                 /* convert to kbits per sec */
20539                 val *= 8;
20540                 val /= 1000;
20541                 optval = (uint32_t)val;
20542                 break;
20543         case TCP_RACK_FORCE_MSEG:
20544                 optval = rack->rc_force_max_seg;
20545                 break;
20546         case TCP_RACK_PACE_MAX_SEG:
20547                 /* Max segments in a pace */
20548                 optval = rack->rc_user_set_max_segs;
20549                 break;
20550         case TCP_RACK_PACE_ALWAYS:
20551                 /* Use the always pace method */
20552                 optval = rack->rc_always_pace;
20553                 break;
20554         case TCP_RACK_PRR_SENDALOT:
20555                 /* Allow PRR to send more than one seg */
20556                 optval = rack->r_ctl.rc_prr_sendalot;
20557                 break;
20558         case TCP_RACK_MIN_TO:
20559                 /* Minimum time between rack t-o's in ms */
20560                 optval = rack->r_ctl.rc_min_to;
20561                 break;
20562         case TCP_RACK_EARLY_SEG:
20563                 /* If early recovery max segments */
20564                 optval = rack->r_ctl.rc_early_recovery_segs;
20565                 break;
20566         case TCP_RACK_REORD_THRESH:
20567                 /* RACK reorder threshold (shift amount) */
20568                 optval = rack->r_ctl.rc_reorder_shift;
20569                 break;
20570         case TCP_RACK_REORD_FADE:
20571                 /* Does reordering fade after ms time */
20572                 optval = rack->r_ctl.rc_reorder_fade;
20573                 break;
20574         case TCP_BBR_USE_RACK_RR:
20575                 /* Do we use the rack cheat for rxt */
20576                 optval = rack->use_rack_rr;
20577                 break;
20578         case TCP_RACK_RR_CONF:
20579                 optval = rack->r_rr_config;
20580                 break;
20581         case TCP_HDWR_RATE_CAP:
20582                 optval = rack->r_rack_hw_rate_caps;
20583                 break;
20584         case TCP_BBR_HDWR_PACE:
20585                 optval = rack->rack_hdw_pace_ena;
20586                 break;
20587         case TCP_RACK_TLP_THRESH:
20588                 /* RACK TLP theshold i.e. srtt+(srtt/N) */
20589                 optval = rack->r_ctl.rc_tlp_threshold;
20590                 break;
20591         case TCP_RACK_PKT_DELAY:
20592                 /* RACK added ms i.e. rack-rtt + reord + N */
20593                 optval = rack->r_ctl.rc_pkt_delay;
20594                 break;
20595         case TCP_RACK_TLP_USE:
20596                 optval = rack->rack_tlp_threshold_use;
20597                 break;
20598         case TCP_RACK_PACE_RATE_CA:
20599                 optval = rack->r_ctl.rc_fixed_pacing_rate_ca;
20600                 break;
20601         case TCP_RACK_PACE_RATE_SS:
20602                 optval = rack->r_ctl.rc_fixed_pacing_rate_ss;
20603                 break;
20604         case TCP_RACK_PACE_RATE_REC:
20605                 optval = rack->r_ctl.rc_fixed_pacing_rate_rec;
20606                 break;
20607         case TCP_RACK_GP_INCREASE_SS:
20608                 optval = rack->r_ctl.rack_per_of_gp_ca;
20609                 break;
20610         case TCP_RACK_GP_INCREASE_CA:
20611                 optval = rack->r_ctl.rack_per_of_gp_ss;
20612                 break;
20613         case TCP_BBR_RACK_RTT_USE:
20614                 optval = rack->r_ctl.rc_rate_sample_method;
20615                 break;
20616         case TCP_DELACK:
20617                 optval = tp->t_delayed_ack;
20618                 break;
20619         case TCP_DATA_AFTER_CLOSE:
20620                 optval = rack->rc_allow_data_af_clo;
20621                 break;
20622         case TCP_SHARED_CWND_TIME_LIMIT:
20623                 optval = rack->r_limit_scw;
20624                 break;
20625         case TCP_RACK_TIMER_SLOP:
20626                 optval = rack->r_ctl.timer_slop;
20627                 break;
20628         default:
20629                 return (tcp_default_ctloutput(inp, sopt));
20630                 break;
20631         }
20632         INP_WUNLOCK(inp);
20633         if (error == 0) {
20634                 if (TCP_PACING_RATE_CAP)
20635                         error = sooptcopyout(sopt, &loptval, sizeof loptval);
20636                 else
20637                         error = sooptcopyout(sopt, &optval, sizeof optval);
20638         }
20639         return (error);
20640 }
20641
20642 static int
20643 rack_ctloutput(struct inpcb *inp, struct sockopt *sopt)
20644 {
20645         if (sopt->sopt_dir == SOPT_SET) {
20646                 return (rack_set_sockopt(inp, sopt));
20647         } else if (sopt->sopt_dir == SOPT_GET) {
20648                 return (rack_get_sockopt(inp, sopt));
20649         } else {
20650                 panic("%s: sopt_dir $%d", __func__, sopt->sopt_dir);
20651         }
20652 }
20653
20654 static const char *rack_stack_names[] = {
20655         __XSTRING(STACKNAME),
20656 #ifdef STACKALIAS
20657         __XSTRING(STACKALIAS),
20658 #endif
20659 };
20660
20661 static int
20662 rack_ctor(void *mem, int32_t size, void *arg, int32_t how)
20663 {
20664         memset(mem, 0, size);
20665         return (0);
20666 }
20667
20668 static void
20669 rack_dtor(void *mem, int32_t size, void *arg)
20670 {
20671
20672 }
20673
20674 static bool rack_mod_inited = false;
20675
20676 static int
20677 tcp_addrack(module_t mod, int32_t type, void *data)
20678 {
20679         int32_t err = 0;
20680         int num_stacks;
20681
20682         switch (type) {
20683         case MOD_LOAD:
20684                 rack_zone = uma_zcreate(__XSTRING(MODNAME) "_map",
20685                     sizeof(struct rack_sendmap),
20686                     rack_ctor, rack_dtor, NULL, NULL, UMA_ALIGN_PTR, 0);
20687
20688                 rack_pcb_zone = uma_zcreate(__XSTRING(MODNAME) "_pcb",
20689                     sizeof(struct tcp_rack),
20690                     rack_ctor, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0);
20691
20692                 sysctl_ctx_init(&rack_sysctl_ctx);
20693                 rack_sysctl_root = SYSCTL_ADD_NODE(&rack_sysctl_ctx,
20694                     SYSCTL_STATIC_CHILDREN(_net_inet_tcp),
20695                     OID_AUTO,
20696 #ifdef STACKALIAS
20697                     __XSTRING(STACKALIAS),
20698 #else
20699                     __XSTRING(STACKNAME),
20700 #endif
20701                     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
20702                     "");
20703                 if (rack_sysctl_root == NULL) {
20704                         printf("Failed to add sysctl node\n");
20705                         err = EFAULT;
20706                         goto free_uma;
20707                 }
20708                 rack_init_sysctls();
20709                 num_stacks = nitems(rack_stack_names);
20710                 err = register_tcp_functions_as_names(&__tcp_rack, M_WAITOK,
20711                     rack_stack_names, &num_stacks);
20712                 if (err) {
20713                         printf("Failed to register %s stack name for "
20714                             "%s module\n", rack_stack_names[num_stacks],
20715                             __XSTRING(MODNAME));
20716                         sysctl_ctx_free(&rack_sysctl_ctx);
20717 free_uma:
20718                         uma_zdestroy(rack_zone);
20719                         uma_zdestroy(rack_pcb_zone);
20720                         rack_counter_destroy();
20721                         printf("Failed to register rack module -- err:%d\n", err);
20722                         return (err);
20723                 }
20724                 tcp_lro_reg_mbufq();
20725                 rack_mod_inited = true;
20726                 break;
20727         case MOD_QUIESCE:
20728                 err = deregister_tcp_functions(&__tcp_rack, true, false);
20729                 break;
20730         case MOD_UNLOAD:
20731                 err = deregister_tcp_functions(&__tcp_rack, false, true);
20732                 if (err == EBUSY)
20733                         break;
20734                 if (rack_mod_inited) {
20735                         uma_zdestroy(rack_zone);
20736                         uma_zdestroy(rack_pcb_zone);
20737                         sysctl_ctx_free(&rack_sysctl_ctx);
20738                         rack_counter_destroy();
20739                         rack_mod_inited = false;
20740                 }
20741                 tcp_lro_dereg_mbufq();
20742                 err = 0;
20743                 break;
20744         default:
20745                 return (EOPNOTSUPP);
20746         }
20747         return (err);
20748 }
20749
20750 static moduledata_t tcp_rack = {
20751         .name = __XSTRING(MODNAME),
20752         .evhand = tcp_addrack,
20753         .priv = 0
20754 };
20755
20756 MODULE_VERSION(MODNAME, 1);
20757 DECLARE_MODULE(MODNAME, tcp_rack, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY);
20758 MODULE_DEPEND(MODNAME, tcphpts, 1, 1, 1);