sys/netinet/tcp_stacks/bbr.c

   1 /*-
   2  * Copyright (c) 2016-9
   3  *      Netflix Inc.
   4  *      All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  */
  28 /**
  29  * Author: Randall Stewart <rrs@netflix.com>
  30  * This work is based on the ACM Queue paper
  31  * BBR - Congestion Based Congestion Control
  32  * and also numerous discussions with Neal, Yuchung and Van.
  33  */
  34
  35 #include <sys/cdefs.h>
  36 __FBSDID("$FreeBSD$");
  37
  38 #include "opt_inet.h"
  39 #include "opt_inet6.h"
  40 #include "opt_ipsec.h"
  41 #include "opt_tcpdebug.h"
  42 #include "opt_ratelimit.h"
  43 #include "opt_kern_tls.h"
  44 #include <sys/param.h>
  45 #include <sys/arb.h>
  46 #include <sys/module.h>
  47 #include <sys/kernel.h>
  48 #ifdef TCP_HHOOK
  49 #include <sys/hhook.h>
  50 #endif
  51 #include <sys/malloc.h>
  52 #include <sys/mbuf.h>
  53 #include <sys/proc.h>
  54 #include <sys/socket.h>
  55 #include <sys/socketvar.h>
  56 #ifdef KERN_TLS
  57 #include <sys/ktls.h>
  58 #endif
  59 #include <sys/sysctl.h>
  60 #include <sys/systm.h>
  61 #ifdef STATS
  62 #include <sys/qmath.h>
  63 #include <sys/tree.h>
  64 #include <sys/stats.h> /* Must come after qmath.h and tree.h */
  65 #endif
  66 #include <sys/refcount.h>
  67 #include <sys/queue.h>
  68 #include <sys/eventhandler.h>
  69 #include <sys/smp.h>
  70 #include <sys/kthread.h>
  71 #include <sys/lock.h>
  72 #include <sys/mutex.h>
  73 #include <sys/tim_filter.h>
  74 #include <sys/time.h>
  75 #include <vm/uma.h>
  76 #include <sys/kern_prefetch.h>
  77
  78 #include <net/route.h>
  79 #include <net/vnet.h>
  80
  81 #define TCPSTATES               /* for logging */
  82
  83 #include <netinet/in.h>
  84 #include <netinet/in_kdtrace.h>
  85 #include <netinet/in_pcb.h>
  86 #include <netinet/ip.h>
  87 #include <netinet/ip_icmp.h>    /* required for icmp_var.h */
  88 #include <netinet/icmp_var.h>   /* for ICMP_BANDLIM */
  89 #include <netinet/ip_var.h>
  90 #include <netinet/ip6.h>
  91 #include <netinet6/in6_pcb.h>
  92 #include <netinet6/ip6_var.h>
  93 #define TCPOUTFLAGS
  94 #include <netinet/tcp.h>
  95 #include <netinet/tcp_fsm.h>
  96 #include <netinet/tcp_seq.h>
  97 #include <netinet/tcp_timer.h>
  98 #include <netinet/tcp_var.h>
  99 #include <netinet/tcpip.h>
 100 #include <netinet/tcp_hpts.h>
 101 #include <netinet/cc/cc.h>
 102 #include <netinet/tcp_log_buf.h>
 103 #include <netinet/tcp_ratelimit.h>
 104 #include <netinet/tcp_lro.h>
 105 #ifdef TCPDEBUG
 106 #include <netinet/tcp_debug.h>
 107 #endif                          /* TCPDEBUG */
 108 #ifdef TCP_OFFLOAD
 109 #include <netinet/tcp_offload.h>
 110 #endif
 111 #ifdef INET6
 112 #include <netinet6/tcp6_var.h>
 113 #endif
 114 #include <netinet/tcp_fastopen.h>
 115
 116 #include <netipsec/ipsec_support.h>
 117 #include <net/if.h>
 118 #include <net/if_var.h>
 119 #include <net/ethernet.h>
 120
 121 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
 122 #include <netipsec/ipsec.h>
 123 #include <netipsec/ipsec6.h>
 124 #endif                          /* IPSEC */
 125
 126 #include <netinet/udp.h>
 127 #include <netinet/udp_var.h>
 128 #include <machine/in_cksum.h>
 129
 130 #ifdef MAC
 131 #include <security/mac/mac_framework.h>
 132 #endif
 133
 134 #include "sack_filter.h"
 135 #include "tcp_bbr.h"
 136 #include "rack_bbr_common.h"
 137 uma_zone_t bbr_zone;
 138 uma_zone_t bbr_pcb_zone;
 139
 140 struct sysctl_ctx_list bbr_sysctl_ctx;
 141 struct sysctl_oid *bbr_sysctl_root;
 142
 143 #define TCPT_RANGESET_NOSLOP(tv, value, tvmin, tvmax) do { \
 144         (tv) = (value); \
 145         if ((u_long)(tv) < (u_long)(tvmin)) \
 146                 (tv) = (tvmin); \
 147         if ((u_long)(tv) > (u_long)(tvmax)) \
 148                 (tv) = (tvmax); \
 149 } while(0)
 150
 151 /*#define BBR_INVARIANT 1*/
 152
 153 /*
 154  * initial window
 155  */
 156 static uint32_t bbr_def_init_win = 10;
 157 static int32_t bbr_persist_min = 250000;        /* 250ms */
 158 static int32_t bbr_persist_max = 1000000;       /* 1 Second */
 159 static int32_t bbr_cwnd_may_shrink = 0;
 160 static int32_t bbr_cwndtarget_rtt_touse = BBR_RTT_PROP;
 161 static int32_t bbr_num_pktepo_for_del_limit = BBR_NUM_RTTS_FOR_DEL_LIMIT;
 162 static int32_t bbr_hardware_pacing_limit = 8000;
 163 static int32_t bbr_quanta = 3;  /* How much extra quanta do we get? */
 164 static int32_t bbr_no_retran = 0;
 165
 166
 167 static int32_t bbr_error_base_paceout = 10000; /* usec to pace */
 168 static int32_t bbr_max_net_error_cnt = 10;
 169 /* Should the following be dynamic too -- loss wise */
 170 static int32_t bbr_rtt_gain_thresh = 0;
 171 /* Measurement controls */
 172 static int32_t bbr_use_google_algo = 1;
 173 static int32_t bbr_ts_limiting = 1;
 174 static int32_t bbr_ts_can_raise = 0;
 175 static int32_t bbr_do_red = 600;
 176 static int32_t bbr_red_scale = 20000;
 177 static int32_t bbr_red_mul = 1;
 178 static int32_t bbr_red_div = 2;
 179 static int32_t bbr_red_growth_restrict = 1;
 180 static int32_t  bbr_target_is_bbunit = 0;
 181 static int32_t bbr_drop_limit = 0;
 182 /*
 183  * How much gain do we need to see to
 184  * stay in startup?
 185  */
 186 static int32_t bbr_marks_rxt_sack_passed = 0;
 187 static int32_t bbr_start_exit = 25;
 188 static int32_t bbr_low_start_exit = 25; /* When we are in reduced gain */
 189 static int32_t bbr_startup_loss_thresh = 2000;  /* 20.00% loss */
 190 static int32_t bbr_hptsi_max_mul = 1;   /* These two mul/div assure a min pacing */
 191 static int32_t bbr_hptsi_max_div = 2;   /* time, 0 means turned off. We need this
 192                                          * if we go back ever to where the pacer
 193                                          * has priority over timers.
 194                                          */
 195 static int32_t bbr_policer_call_from_rack_to = 0;
 196 static int32_t bbr_policer_detection_enabled = 1;
 197 static int32_t bbr_min_measurements_req = 1;    /* We need at least 2
 198                                                  * measurments before we are
 199                                                  * "good" note that 2 == 1.
 200                                                  * This is because we use a >
 201                                                  * comparison. This means if
 202                                                  * min_measure was 0, it takes
 203                                                  * num-measures > min(0) and
 204                                                  * you get 1 measurement and
 205                                                  * you are good. Set to 1, you
 206                                                  * have to have two
 207                                                  * measurements (this is done
 208                                                  * to prevent it from being ok
 209                                                  * to have no measurements). */
 210 static int32_t bbr_no_pacing_until = 4;
 211
 212 static int32_t bbr_min_usec_delta = 20000;      /* 20,000 usecs */
 213 static int32_t bbr_min_peer_delta = 20;         /* 20 units */
 214 static int32_t bbr_delta_percent = 150;         /* 15.0 % */
 215
 216 static int32_t bbr_target_cwnd_mult_limit = 8;
 217 /*
 218  * bbr_cwnd_min_val is the number of
 219  * segments we hold to in the RTT probe
 220  * state typically 4.
 221  */
 222 static int32_t bbr_cwnd_min_val = BBR_PROBERTT_NUM_MSS;
 223
 224
 225 static int32_t bbr_cwnd_min_val_hs = BBR_HIGHSPEED_NUM_MSS;
 226
 227 static int32_t bbr_gain_to_target = 1;
 228 static int32_t bbr_gain_gets_extra_too = 1;
 229 /*
 230  * bbr_high_gain is the 2/ln(2) value we need
 231  * to double the sending rate in startup. This
 232  * is used for both cwnd and hptsi gain's.
 233  */
 234 static int32_t bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1;
 235 static int32_t bbr_startup_lower = BBR_UNIT * 1500 / 1000 + 1;
 236 static int32_t bbr_use_lower_gain_in_startup = 1;
 237
 238 /* thresholds for reduction on drain in sub-states/drain */
 239 static int32_t bbr_drain_rtt = BBR_SRTT;
 240 static int32_t bbr_drain_floor = 88;
 241 static int32_t google_allow_early_out = 1;
 242 static int32_t google_consider_lost = 1;
 243 static int32_t bbr_drain_drop_mul = 4;
 244 static int32_t bbr_drain_drop_div = 5;
 245 static int32_t bbr_rand_ot = 50;
 246 static int32_t bbr_can_force_probertt = 0;
 247 static int32_t bbr_can_adjust_probertt = 1;
 248 static int32_t bbr_probertt_sets_rtt = 0;
 249 static int32_t bbr_can_use_ts_for_rtt = 1;
 250 static int32_t bbr_is_ratio = 0;
 251 static int32_t bbr_sub_drain_app_limit = 1;
 252 static int32_t bbr_prtt_slam_cwnd = 1;
 253 static int32_t bbr_sub_drain_slam_cwnd = 1;
 254 static int32_t bbr_slam_cwnd_in_main_drain = 1;
 255 static int32_t bbr_filter_len_sec = 6;  /* How long does the rttProp filter
 256                                          * hold */
 257 static uint32_t bbr_rtt_probe_limit = (USECS_IN_SECOND * 4);
 258 /*
 259  * bbr_drain_gain is the reverse of the high_gain
 260  * designed to drain back out the standing queue
 261  * that is formed in startup by causing a larger
 262  * hptsi gain and thus drainging the packets
 263  * in flight.
 264  */
 265 static int32_t bbr_drain_gain = BBR_UNIT * 1000 / 2885;
 266 static int32_t bbr_rttprobe_gain = 192;
 267
 268 /*
 269  * The cwnd_gain is the default cwnd gain applied when
 270  * calculating a target cwnd. Note that the cwnd is
 271  * a secondary factor in the way BBR works (see the
 272  * paper and think about it, it will take some time).
 273  * Basically the hptsi_gain spreads the packets out
 274  * so you never get more than BDP to the peer even
 275  * if the cwnd is high. In our implemenation that
 276  * means in non-recovery/retransmission scenarios
 277  * cwnd will never be reached by the flight-size.
 278  */
 279 static int32_t bbr_cwnd_gain = BBR_UNIT * 2;
 280 static int32_t bbr_tlp_type_to_use = BBR_SRTT;
 281 static int32_t bbr_delack_time = 100000;        /* 100ms in useconds */
 282 static int32_t bbr_sack_not_required = 0;       /* set to one to allow non-sack to use bbr */
 283 static int32_t bbr_initial_bw_bps = 62500;      /* 500kbps in bytes ps */
 284 static int32_t bbr_ignore_data_after_close = 1;
 285 static int16_t bbr_hptsi_gain[] = {
 286         (BBR_UNIT *5 / 4),
 287         (BBR_UNIT * 3 / 4),
 288         BBR_UNIT,
 289         BBR_UNIT,
 290         BBR_UNIT,
 291         BBR_UNIT,
 292         BBR_UNIT,
 293         BBR_UNIT
 294 };
 295 int32_t bbr_use_rack_resend_cheat = 1;
 296 int32_t bbr_sends_full_iwnd = 1;
 297
 298 #define BBR_HPTSI_GAIN_MAX 8
 299 /*
 300  * The BBR module incorporates a number of
 301  * TCP ideas that have been put out into the IETF
 302  * over the last few years:
 303  * - Yuchung Cheng's RACK TCP (for which its named) that
 304  *    will stop us using the number of dup acks and instead
 305  *    use time as the gage of when we retransmit.
 306  * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
 307  *    of Dukkipati et.al.
 308  * - Van Jacobson's et.al BBR.
 309  *
 310  * RACK depends on SACK, so if an endpoint arrives that
 311  * cannot do SACK the state machine below will shuttle the
 312  * connection back to using the "default" TCP stack that is
 313  * in FreeBSD.
 314  *
 315  * To implement BBR and RACK the original TCP stack was first decomposed
 316  * into a functional state machine with individual states
 317  * for each of the possible TCP connection states. The do_segement
 318  * functions role in life is to mandate the connection supports SACK
 319  * initially and then assure that the RACK state matches the conenction
 320  * state before calling the states do_segment function. Data processing
 321  * of inbound segments also now happens in the hpts_do_segment in general
 322  * with only one exception. This is so we can keep the connection on
 323  * a single CPU.
 324  *
 325  * Each state is simplified due to the fact that the original do_segment
 326  * has been decomposed and we *know* what state we are in (no
 327  * switches on the state) and all tests for SACK are gone. This
 328  * greatly simplifies what each state does.
 329  *
 330  * TCP output is also over-written with a new version since it
 331  * must maintain the new rack scoreboard and has had hptsi
 332  * integrated as a requirment. Still todo is to eliminate the
 333  * use of the callout_() system and use the hpts for all
 334  * timers as well.
 335  */
 336 static uint32_t bbr_rtt_probe_time = 200000;    /* 200ms in micro seconds */
 337 static uint32_t bbr_rtt_probe_cwndtarg = 4;     /* How many mss's outstanding */
 338 static const int32_t bbr_min_req_free = 2;      /* The min we must have on the
 339                                                  * free list */
 340 static int32_t bbr_tlp_thresh = 1;
 341 static int32_t bbr_reorder_thresh = 2;
 342 static int32_t bbr_reorder_fade = 60000000;     /* 0 - never fade, def
 343                                                  * 60,000,000 - 60 seconds */
 344 static int32_t bbr_pkt_delay = 1000;
 345 static int32_t bbr_min_to = 1000;       /* Number of usec's minimum timeout */
 346 static int32_t bbr_incr_timers = 1;
 347
 348 static int32_t bbr_tlp_min = 10000;     /* 10ms in usecs */
 349 static int32_t bbr_delayed_ack_time = 200000;   /* 200ms in usecs */
 350 static int32_t bbr_exit_startup_at_loss = 1;
 351
 352 /*
 353  * bbr_lt_bw_ratio is 1/8th
 354  * bbr_lt_bw_diff is  < 4 Kbit/sec
 355  */
 356 static uint64_t bbr_lt_bw_diff = 4000 / 8;      /* In bytes per second */
 357 static uint64_t bbr_lt_bw_ratio = 8;    /* For 1/8th */
 358 static uint32_t bbr_lt_bw_max_rtts = 48;        /* How many rtt's do we use
 359                                                  * the lt_bw for */
 360 static uint32_t bbr_lt_intvl_min_rtts = 4;      /* Min num of RTT's to measure
 361                                                  * lt_bw */
 362 static int32_t bbr_lt_intvl_fp = 0;             /* False positive epoch diff */
 363 static int32_t bbr_lt_loss_thresh = 196;        /* Lost vs delivered % */
 364 static int32_t bbr_lt_fd_thresh = 100;          /* false detection % */
 365
 366 static int32_t bbr_verbose_logging = 0;
 367 /*
 368  * Currently regular tcp has a rto_min of 30ms
 369  * the backoff goes 12 times so that ends up
 370  * being a total of 122.850 seconds before a
 371  * connection is killed.
 372  */
 373 static int32_t bbr_rto_min_ms = 30;     /* 30ms same as main freebsd */
 374 static int32_t bbr_rto_max_sec = 4;     /* 4 seconds */
 375
 376 /****************************************************/
 377 /* DEFAULT TSO SIZING  (cpu performance impacting)  */
 378 /****************************************************/
 379 /* What amount is our formula using to get TSO size */
 380 static int32_t bbr_hptsi_per_second = 1000;
 381
 382 /*
 383  * For hptsi under bbr_cross_over connections what is delay
 384  * target 7ms (in usec) combined with a seg_max of 2
 385  * gets us close to identical google behavior in
 386  * TSO size selection (possibly more 1MSS sends).
 387  */
 388 static int32_t bbr_hptsi_segments_delay_tar = 7000;
 389
 390 /* Does pacing delay include overhead's in its time calculations? */
 391 static int32_t bbr_include_enet_oh = 0;
 392 static int32_t bbr_include_ip_oh = 1;
 393 static int32_t bbr_include_tcp_oh = 1;
 394 static int32_t bbr_google_discount = 10;
 395
 396 /* Do we use (nf mode) pkt-epoch to drive us or rttProp? */
 397 static int32_t bbr_state_is_pkt_epoch = 0;
 398 static int32_t bbr_state_drain_2_tar = 1;
 399 /* What is the max the 0 - bbr_cross_over MBPS TSO target
 400  * can reach using our delay target. Note that this
 401  * value becomes the floor for the cross over
 402  * algorithm.
 403  */
 404 static int32_t bbr_hptsi_segments_max = 2;
 405 static int32_t bbr_hptsi_segments_floor = 1;
 406 static int32_t bbr_hptsi_utter_max = 0;
 407
 408 /* What is the min the 0 - bbr_cross-over MBPS  TSO target can be */
 409 static int32_t bbr_hptsi_bytes_min = 1460;
 410 static int32_t bbr_all_get_min = 0;
 411
 412 /* Cross over point from algo-a to algo-b */
 413 static uint32_t bbr_cross_over = TWENTY_THREE_MBPS;
 414
 415 /* Do we deal with our restart state? */
 416 static int32_t bbr_uses_idle_restart = 0;
 417 static int32_t bbr_idle_restart_threshold = 100000;     /* 100ms in useconds */
 418
 419 /* Do we allow hardware pacing? */
 420 static int32_t bbr_allow_hdwr_pacing = 0;
 421 static int32_t bbr_hdwr_pace_adjust = 2;        /* multipler when we calc the tso size */
 422 static int32_t bbr_hdwr_pace_floor = 1;
 423 static int32_t bbr_hdwr_pacing_delay_cnt = 10;
 424
 425 /****************************************************/
 426 static int32_t bbr_resends_use_tso = 0;
 427 static int32_t bbr_tlp_max_resend = 2;
 428 static int32_t bbr_sack_block_limit = 128;
 429
 430 #define  BBR_MAX_STAT 19
 431 counter_u64_t bbr_state_time[BBR_MAX_STAT];
 432 counter_u64_t bbr_state_lost[BBR_MAX_STAT];
 433 counter_u64_t bbr_state_resend[BBR_MAX_STAT];
 434 counter_u64_t bbr_stat_arry[BBR_STAT_SIZE];
 435 counter_u64_t bbr_opts_arry[BBR_OPTS_SIZE];
 436 counter_u64_t bbr_out_size[TCP_MSS_ACCT_SIZE];
 437 counter_u64_t bbr_flows_whdwr_pacing;
 438 counter_u64_t bbr_flows_nohdwr_pacing;
 439
 440 counter_u64_t bbr_nohdwr_pacing_enobuf;
 441 counter_u64_t bbr_hdwr_pacing_enobuf;
 442
 443 static inline uint64_t bbr_get_bw(struct tcp_bbr *bbr);
 444
 445 /*
 446  * Static defintions we need for forward declarations.
 447  */
 448 static uint32_t
 449 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain,
 450     uint32_t useconds_time, uint64_t bw);
 451 static uint32_t
 452 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain);
 453 static void
 454      bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win);
 455 static void
 456 bbr_set_probebw_gains(struct tcp_bbr *bbr,  uint32_t cts, uint32_t losses);
 457 static void
 458 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int line,
 459                     int dolog);
 460 static uint32_t
 461 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain);
 462 static void
 463 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch,
 464                  int32_t pkt_epoch, uint32_t losses);
 465 static uint32_t
 466 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm);
 467 static uint32_t bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp);
 468 static uint32_t
 469 bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr,
 470     struct bbr_sendmap *rsm, uint32_t srtt,
 471     uint32_t cts);
 472 static void
 473 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts,
 474     int32_t line);
 475 static void
 476      bbr_set_state_target(struct tcp_bbr *bbr, int line);
 477 static void
 478      bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line);
 479
 480 static void
 481      bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line);
 482
 483 static void
 484      tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts);
 485
 486 static void
 487      bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts);
 488
 489 static void
 490      bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied, uint32_t rtt,
 491                          uint32_t line, uint8_t is_start, uint16_t set);
 492
 493 static struct bbr_sendmap *
 494             bbr_find_lowest_rsm(struct tcp_bbr *bbr);
 495 static __inline uint32_t
 496 bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type);
 497 static void
 498      bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which);
 499
 500 static void
 501 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt,
 502     uint32_t thresh, uint32_t to);
 503 static void
 504      bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag);
 505
 506 static void
 507 bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot,
 508     uint32_t del_by, uint32_t cts, uint32_t sloton, uint32_t prev_delay);
 509
 510 static void
 511 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr,
 512     uint32_t cts, int32_t line);
 513 static void
 514      bbr_stop_all_timers(struct tcpcb *tp);
 515 static void
 516      bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts);
 517 static void
 518      bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts);
 519 static void
 520      bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts);
 521
 522
 523 static void
 524 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
 525     uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod);
 526
 527 static inline uint8_t
 528 bbr_state_val(struct tcp_bbr *bbr)
 529 {
 530         return(bbr->rc_bbr_substate);
 531 }
 532
 533 static inline uint32_t
 534 get_min_cwnd(struct tcp_bbr *bbr)
 535 {
 536         int mss;
 537
 538         mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
 539         if (bbr_get_rtt(bbr, BBR_RTT_PROP) < BBR_HIGH_SPEED)
 540                 return (bbr_cwnd_min_val_hs * mss);
 541         else
 542                 return (bbr_cwnd_min_val * mss);
 543 }
 544
 545 static uint32_t
 546 bbr_get_persists_timer_val(struct tcpcb *tp, struct tcp_bbr *bbr)
 547 {
 548         uint64_t srtt, var;
 549         uint64_t ret_val;
 550
 551         bbr->r_ctl.rc_hpts_flags |= PACE_TMR_PERSIT;
 552         if (tp->t_srtt == 0) {
 553                 srtt = (uint64_t)BBR_INITIAL_RTO;
 554                 var = 0;
 555         } else {
 556                 srtt = ((uint64_t)TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);
 557                 var = ((uint64_t)TICKS_2_USEC(tp->t_rttvar) >> TCP_RTT_SHIFT);
 558         }
 559         TCPT_RANGESET_NOSLOP(ret_val, ((srtt + var) * tcp_backoff[tp->t_rxtshift]),
 560             bbr_persist_min, bbr_persist_max);
 561         return ((uint32_t)ret_val);
 562 }
 563
 564 static uint32_t
 565 bbr_timer_start(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
 566 {
 567         /*
 568          * Start the FR timer, we do this based on getting the first one in
 569          * the rc_tmap. Note that if its NULL we must stop the timer. in all
 570          * events we need to stop the running timer (if its running) before
 571          * starting the new one.
 572          */
 573         uint32_t thresh, exp, to, srtt, time_since_sent, tstmp_touse;
 574         int32_t idx;
 575         int32_t is_tlp_timer = 0;
 576         struct bbr_sendmap *rsm;
 577
 578         if (bbr->rc_all_timers_stopped) {
 579                 /* All timers have been stopped none are to run */
 580                 return (0);
 581         }
 582         if (bbr->rc_in_persist) {
 583                 /* We can't start any timer in persists */
 584                 return (bbr_get_persists_timer_val(tp, bbr));
 585         }
 586         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
 587         if ((rsm == NULL) ||
 588             ((tp->t_flags & TF_SACK_PERMIT) == 0) ||
 589             (tp->t_state < TCPS_ESTABLISHED)) {
 590                 /* Nothing on the send map */
 591 activate_rxt:
 592                 if (SEQ_LT(tp->snd_una, tp->snd_max) || sbavail(&(tp->t_inpcb->inp_socket->so_snd))) {
 593                         uint64_t tov;
 594
 595                         time_since_sent = 0;
 596                         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
 597                         if (rsm) {
 598                                 idx = rsm->r_rtr_cnt - 1;
 599                                 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
 600                                         tstmp_touse = rsm->r_tim_lastsent[idx];
 601                                 else
 602                                         tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
 603                                 if (TSTMP_GT(tstmp_touse, cts))
 604                                     time_since_sent = cts - tstmp_touse;
 605                         }
 606                         bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RXT;
 607                         if (tp->t_srtt == 0)
 608                                 tov = BBR_INITIAL_RTO;
 609                         else
 610                                 tov = ((uint64_t)(TICKS_2_USEC(tp->t_srtt) +
 611                                     ((uint64_t)TICKS_2_USEC(tp->t_rttvar) * (uint64_t)4)) >> TCP_RTT_SHIFT);
 612                         if (tp->t_rxtshift)
 613                                 tov *= tcp_backoff[tp->t_rxtshift];
 614                         if (tov > time_since_sent)
 615                                 tov -= time_since_sent;
 616                         else
 617                                 tov = bbr->r_ctl.rc_min_to;
 618                         TCPT_RANGESET_NOSLOP(to, tov,
 619                             (bbr->r_ctl.rc_min_rto_ms * MS_IN_USEC),
 620                             (bbr->rc_max_rto_sec * USECS_IN_SECOND));
 621                         bbr_log_timer_var(bbr, 2, cts, 0, srtt, 0, to);
 622                         return (to);
 623                 }
 624                 return (0);
 625         }
 626         if (rsm->r_flags & BBR_ACKED) {
 627                 rsm = bbr_find_lowest_rsm(bbr);
 628                 if (rsm == NULL) {
 629                         /* No lowest? */
 630                         goto activate_rxt;
 631                 }
 632         }
 633         /* Convert from ms to usecs */
 634         if (rsm->r_flags & BBR_SACK_PASSED) {
 635                 if ((tp->t_flags & TF_SENTFIN) &&
 636                     ((tp->snd_max - tp->snd_una) == 1) &&
 637                     (rsm->r_flags & BBR_HAS_FIN)) {
 638                         /*
 639                          * We don't start a bbr rack timer if all we have is
 640                          * a FIN outstanding.
 641                          */
 642                         goto activate_rxt;
 643                 }
 644                 srtt = bbr_get_rtt(bbr, BBR_RTT_RACK);
 645                 thresh = bbr_calc_thresh_rack(bbr, srtt, cts, rsm);
 646                 idx = rsm->r_rtr_cnt - 1;
 647                 exp = rsm->r_tim_lastsent[idx] + thresh;
 648                 if (SEQ_GEQ(exp, cts)) {
 649                         to = exp - cts;
 650                         if (to < bbr->r_ctl.rc_min_to) {
 651                                 to = bbr->r_ctl.rc_min_to;
 652                         }
 653                 } else {
 654                         to = bbr->r_ctl.rc_min_to;
 655                 }
 656         } else {
 657                 /* Ok we need to do a TLP not RACK */
 658                 if (bbr->rc_tlp_in_progress != 0) {
 659                         /*
 660                          * The previous send was a TLP.
 661                          */
 662                         goto activate_rxt;
 663                 }
 664                 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
 665                 if (rsm == NULL) {
 666                         /* We found no rsm to TLP with. */
 667                         goto activate_rxt;
 668                 }
 669                 if (rsm->r_flags & BBR_HAS_FIN) {
 670                         /* If its a FIN we don't do TLP */
 671                         rsm = NULL;
 672                         goto activate_rxt;
 673                 }
 674                 time_since_sent = 0;
 675                 idx = rsm->r_rtr_cnt - 1;
 676                 if (TSTMP_GEQ(rsm->r_tim_lastsent[idx], bbr->r_ctl.rc_tlp_rxt_last_time))
 677                         tstmp_touse = rsm->r_tim_lastsent[idx];
 678                 else
 679                         tstmp_touse = bbr->r_ctl.rc_tlp_rxt_last_time;
 680                 if (TSTMP_GT(tstmp_touse, cts))
 681                     time_since_sent = cts - tstmp_touse;
 682                 is_tlp_timer = 1;
 683                 srtt = bbr_get_rtt(bbr, bbr_tlp_type_to_use);
 684                 thresh = bbr_calc_thresh_tlp(tp, bbr, rsm, srtt, cts);
 685                 if (thresh > time_since_sent)
 686                         to = thresh - time_since_sent;
 687                 else
 688                         to = bbr->r_ctl.rc_min_to;
 689                 if (to > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
 690                         /*
 691                          * If the TLP time works out to larger than the max
 692                          * RTO lets not do TLP.. just RTO.
 693                          */
 694                         goto activate_rxt;
 695                 }
 696                 if ((bbr->rc_tlp_rtx_out == 1) &&
 697                     (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq)) {
 698                         /*
 699                          * Second retransmit of the same TLP
 700                          * lets not.
 701                          */
 702                         bbr->rc_tlp_rtx_out = 0;
 703                         goto activate_rxt;
 704                 }
 705                 if (rsm->r_start != bbr->r_ctl.rc_last_tlp_seq) {
 706                         /*
 707                          * The tail is no longer the last one I did a probe
 708                          * on
 709                          */
 710                         bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
 711                         bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
 712                 }
 713         }
 714         if (is_tlp_timer == 0) {
 715                 BBR_STAT_INC(bbr_to_arm_rack);
 716                 bbr->r_ctl.rc_hpts_flags |= PACE_TMR_RACK;
 717         } else {
 718                 bbr_log_timer_var(bbr, 1, cts, time_since_sent, srtt, thresh, to);
 719                 if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
 720                         /*
 721                          * We have exceeded how many times we can retran the
 722                          * current TLP timer, switch to the RTO timer.
 723                          */
 724                         goto activate_rxt;
 725                 } else {
 726                         BBR_STAT_INC(bbr_to_arm_tlp);
 727                         bbr->r_ctl.rc_hpts_flags |= PACE_TMR_TLP;
 728                 }
 729         }
 730         return (to);
 731 }
 732
 733 static inline int32_t
 734 bbr_minseg(struct tcp_bbr *bbr)
 735 {
 736         return (bbr->r_ctl.rc_pace_min_segs - bbr->rc_last_options);
 737 }
 738
 739 static void
 740 bbr_start_hpts_timer(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts, int32_t frm, int32_t slot, uint32_t tot_len)
 741 {
 742         struct inpcb *inp;
 743         struct hpts_diag diag;
 744         uint32_t delayed_ack = 0;
 745         uint32_t left = 0;
 746         uint32_t hpts_timeout;
 747         uint8_t stopped;
 748         int32_t delay_calc = 0;
 749         uint32_t prev_delay = 0;
 750
 751         inp = tp->t_inpcb;
 752         if (inp->inp_in_hpts) {
 753                 /* A previous call is already set up */
 754                 return;
 755         }
 756         if ((tp->t_state == TCPS_CLOSED) ||
 757             (tp->t_state == TCPS_LISTEN)) {
 758                 return;
 759         }
 760         stopped = bbr->rc_tmr_stopped;
 761         if (stopped && TSTMP_GT(bbr->r_ctl.rc_timer_exp, cts)) {
 762                 left = bbr->r_ctl.rc_timer_exp - cts;
 763         }
 764         bbr->r_ctl.rc_hpts_flags = 0;
 765         bbr->r_ctl.rc_timer_exp = 0;
 766         prev_delay = bbr->r_ctl.rc_last_delay_val;
 767         if (bbr->r_ctl.rc_last_delay_val &&
 768             (slot == 0)) {
 769                 /*
 770                  * If a previous pacer delay was in place we
 771                  * are not coming from the output side (where
 772                  * we calculate a delay, more likely a timer).
 773                  */
 774                 slot = bbr->r_ctl.rc_last_delay_val;
 775                 if (TSTMP_GT(cts, bbr->rc_pacer_started)) {
 776                         /* Compensate for time passed  */
 777                         delay_calc = cts - bbr->rc_pacer_started;
 778                         if (delay_calc <= slot)
 779                                 slot -= delay_calc;
 780                 }
 781         }
 782         /* Do we have early to make up for by pushing out the pacing time? */
 783         if (bbr->r_agg_early_set) {
 784                 bbr_log_pacing_delay_calc(bbr, 0, bbr->r_ctl.rc_agg_early, cts, slot, 0, bbr->r_agg_early_set, 2);
 785                 slot += bbr->r_ctl.rc_agg_early;
 786                 bbr->r_ctl.rc_agg_early = 0;
 787                 bbr->r_agg_early_set = 0;
 788         }
 789         /* Are we running a total debt that needs to be compensated for? */
 790         if (bbr->r_ctl.rc_hptsi_agg_delay) {
 791                 if (slot > bbr->r_ctl.rc_hptsi_agg_delay) {
 792                         /* We nuke the delay */
 793                         slot -= bbr->r_ctl.rc_hptsi_agg_delay;
 794                         bbr->r_ctl.rc_hptsi_agg_delay = 0;
 795                 } else {
 796                         /* We nuke some of the delay, put in a minimal 100usecs  */
 797                         bbr->r_ctl.rc_hptsi_agg_delay -= slot;
 798                         bbr->r_ctl.rc_last_delay_val = slot = 100;
 799                 }
 800         }
 801         bbr->r_ctl.rc_last_delay_val = slot;
 802         hpts_timeout = bbr_timer_start(tp, bbr, cts);
 803         if (tp->t_flags & TF_DELACK) {
 804                 if (bbr->rc_in_persist == 0) {
 805                         delayed_ack = bbr_delack_time;
 806                 } else {
 807                         /*
 808                          * We are in persists and have
 809                          * gotten a new data element.
 810                          */
 811                         if (hpts_timeout > bbr_delack_time) {
 812                                 /*
 813                                  * Lets make the persists timer (which acks)
 814                                  * be the smaller of hpts_timeout and bbr_delack_time.
 815                                  */
 816                                 hpts_timeout = bbr_delack_time;
 817                         }
 818                 }
 819         }
 820         if (delayed_ack &&
 821             ((hpts_timeout == 0) ||
 822              (delayed_ack < hpts_timeout))) {
 823                 /* We need a Delayed ack timer */
 824                 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
 825                 hpts_timeout = delayed_ack;
 826         }
 827         if (slot) {
 828                 /* Mark that we have a pacing timer up */
 829                 BBR_STAT_INC(bbr_paced_segments);
 830                 bbr->r_ctl.rc_hpts_flags |= PACE_PKT_OUTPUT;
 831         }
 832         /*
 833          * If no timers are going to run and we will fall off thfe hptsi
 834          * wheel, we resort to a keep-alive timer if its configured.
 835          */
 836         if ((hpts_timeout == 0) &&
 837             (slot == 0)) {
 838                 if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
 839                     (tp->t_state <= TCPS_CLOSING)) {
 840                         /*
 841                          * Ok we have no timer (persists, rack, tlp, rxt  or
 842                          * del-ack), we don't have segments being paced. So
 843                          * all that is left is the keepalive timer.
 844                          */
 845                         if (TCPS_HAVEESTABLISHED(tp->t_state)) {
 846                                 hpts_timeout = TICKS_2_USEC(TP_KEEPIDLE(tp));
 847                         } else {
 848                                 hpts_timeout = TICKS_2_USEC(TP_KEEPINIT(tp));
 849                         }
 850                         bbr->r_ctl.rc_hpts_flags |= PACE_TMR_KEEP;
 851                 }
 852         }
 853         if (left && (stopped & (PACE_TMR_KEEP | PACE_TMR_DELACK)) ==
 854             (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK)) {
 855                 /*
 856                  * RACK, TLP, persists and RXT timers all are restartable
 857                  * based on actions input .. i.e we received a packet (ack
 858                  * or sack) and that changes things (rw, or snd_una etc).
 859                  * Thus we can restart them with a new value. For
 860                  * keep-alive, delayed_ack we keep track of what was left
 861                  * and restart the timer with a smaller value.
 862                  */
 863                 if (left < hpts_timeout)
 864                         hpts_timeout = left;
 865         }
 866         if (bbr->r_ctl.rc_incr_tmrs && slot &&
 867             (bbr->r_ctl.rc_hpts_flags & (PACE_TMR_TLP|PACE_TMR_RXT))) {
 868                 /*
 869                  * If configured to do so, and the timer is either
 870                  * the TLP or RXT timer, we need to increase the timeout
 871                  * by the pacing time. Consider the bottleneck at my
 872                  * machine as an example, we are sending something
 873                  * to start a TLP on. The last packet won't be emitted
 874                  * fully until the pacing time (the bottleneck will hold
 875                  * the data in place). Once the packet is emitted that
 876                  * is when we want to start waiting for the TLP. This
 877                  * is most evident with hardware pacing (where the nic
 878                  * is holding the packet(s) before emitting). But it
 879                  * can also show up in the network so we do it for all
 880                  * cases. Technically we would take off one packet from
 881                  * this extra delay but this is easier and being more
 882                  * conservative is probably better.
 883                  */
 884                 hpts_timeout += slot;
 885         }
 886         if (hpts_timeout) {
 887                 /*
 888                  * Hack alert for now we can't time-out over 2147 seconds (a
 889                  * bit more than 35min)
 890                  */
 891                 if (hpts_timeout > 0x7ffffffe)
 892                         hpts_timeout = 0x7ffffffe;
 893                 bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
 894         } else
 895                 bbr->r_ctl.rc_timer_exp = 0;
 896         if ((slot) &&
 897             (bbr->rc_use_google ||
 898              bbr->output_error_seen ||
 899              (slot <= hpts_timeout))  ) {
 900                 /*
 901                  * Tell LRO that it can queue packets while
 902                  * we pace.
 903                  */
 904                 bbr->rc_inp->inp_flags2 |= INP_MBUF_QUEUE_READY;
 905                 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
 906                     (bbr->rc_cwnd_limited == 0)) {
 907                         /*
 908                          * If we are not cwnd limited and we
 909                          * are running a rack timer we put on
 910                          * the do not disturbe even for sack.
 911                          */
 912                         inp->inp_flags2 |= INP_DONT_SACK_QUEUE;
 913                 } else
 914                         inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
 915                 bbr->rc_pacer_started = cts;
 916
 917                 (void)tcp_hpts_insert_diag(tp->t_inpcb, HPTS_USEC_TO_SLOTS(slot),
 918                                            __LINE__, &diag);
 919                 bbr->rc_timer_first = 0;
 920                 bbr->bbr_timer_src = frm;
 921                 bbr_log_to_start(bbr, cts, hpts_timeout, slot, 1);
 922                 bbr_log_hpts_diag(bbr, cts, &diag);
 923         } else if (hpts_timeout) {
 924                 (void)tcp_hpts_insert_diag(tp->t_inpcb, HPTS_USEC_TO_SLOTS(hpts_timeout),
 925                                            __LINE__, &diag);
 926                 /*
 927                  * We add the flag here as well if the slot is set,
 928                  * since hpts will call in to clear the queue first before
 929                  * calling the output routine (which does our timers).
 930                  * We don't want to set the flag if its just a timer
 931                  * else the arrival of data might (that causes us
 932                  * to send more) might get delayed. Imagine being
 933                  * on a keep-alive timer and a request comes in for
 934                  * more data.
 935                  */
 936                 if (slot)
 937                         bbr->rc_pacer_started = cts;
 938                 if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) &&
 939                     (bbr->rc_cwnd_limited == 0)) {
 940                         /*
 941                          * For a rack timer, don't wake us even
 942                          * if a sack arrives as long as we are
 943                          * not cwnd limited.
 944                          */
 945                         bbr->rc_inp->inp_flags2 |= INP_MBUF_QUEUE_READY;
 946                         inp->inp_flags2 |= INP_DONT_SACK_QUEUE;
 947                 } else {
 948                         /* All other timers wake us up */
 949                         bbr->rc_inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
 950                         inp->inp_flags2 &= ~INP_DONT_SACK_QUEUE;
 951                 }
 952                 bbr->bbr_timer_src = frm;
 953                 bbr_log_to_start(bbr, cts, hpts_timeout, slot, 0);
 954                 bbr_log_hpts_diag(bbr, cts, &diag);
 955                 bbr->rc_timer_first = 1;
 956         }
 957         bbr->rc_tmr_stopped = 0;
 958         bbr_log_type_bbrsnd(bbr, tot_len, slot, delay_calc, cts, frm, prev_delay);
 959 }
 960
 961 static void
 962 bbr_timer_audit(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, struct sockbuf *sb)
 963 {
 964         /*
 965          * We received an ack, and then did not call send or were bounced
 966          * out due to the hpts was running. Now a timer is up as well, is it
 967          * the right timer?
 968          */
 969         struct inpcb *inp;
 970         struct bbr_sendmap *rsm;
 971         uint32_t hpts_timeout;
 972         int tmr_up;
 973
 974         tmr_up = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
 975         if (bbr->rc_in_persist && (tmr_up == PACE_TMR_PERSIT))
 976                 return;
 977         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
 978         if (((rsm == NULL) || (tp->t_state < TCPS_ESTABLISHED)) &&
 979             (tmr_up == PACE_TMR_RXT)) {
 980                 /* Should be an RXT */
 981                 return;
 982         }
 983         inp = bbr->rc_inp;
 984         if (rsm == NULL) {
 985                 /* Nothing outstanding? */
 986                 if (tp->t_flags & TF_DELACK) {
 987                         if (tmr_up == PACE_TMR_DELACK)
 988                                 /*
 989                                  * We are supposed to have delayed ack up
 990                                  * and we do
 991                                  */
 992                                 return;
 993                 } else if (sbavail(&inp->inp_socket->so_snd) &&
 994                     (tmr_up == PACE_TMR_RXT)) {
 995                         /*
 996                          * if we hit enobufs then we would expect the
 997                          * possiblity of nothing outstanding and the RXT up
 998                          * (and the hptsi timer).
 999                          */
1000                         return;
1001                 } else if (((V_tcp_always_keepalive ||
1002                             inp->inp_socket->so_options & SO_KEEPALIVE) &&
1003                             (tp->t_state <= TCPS_CLOSING)) &&
1004                             (tmr_up == PACE_TMR_KEEP) &&
1005                     (tp->snd_max == tp->snd_una)) {
1006                         /* We should have keep alive up and we do */
1007                         return;
1008                 }
1009         }
1010         if (rsm && (rsm->r_flags & BBR_SACK_PASSED)) {
1011                 if ((tp->t_flags & TF_SENTFIN) &&
1012                     ((tp->snd_max - tp->snd_una) == 1) &&
1013                     (rsm->r_flags & BBR_HAS_FIN)) {
1014                         /* needs to be a RXT */
1015                         if (tmr_up == PACE_TMR_RXT)
1016                                 return;
1017                         else
1018                                 goto wrong_timer;
1019                 } else if (tmr_up == PACE_TMR_RACK)
1020                         return;
1021                 else
1022                         goto wrong_timer;
1023         } else if (rsm && (tmr_up == PACE_TMR_RACK)) {
1024                 /* Rack timer has priority if we have data out */
1025                 return;
1026         } else if (SEQ_GT(tp->snd_max, tp->snd_una) &&
1027                     ((tmr_up == PACE_TMR_TLP) ||
1028             (tmr_up == PACE_TMR_RXT))) {
1029                 /*
1030                  * Either a TLP or RXT is fine if no sack-passed is in place
1031                  * and data is outstanding.
1032                  */
1033                 return;
1034         } else if (tmr_up == PACE_TMR_DELACK) {
1035                 /*
1036                  * If the delayed ack was going to go off before the
1037                  * rtx/tlp/rack timer were going to expire, then that would
1038                  * be the timer in control. Note we don't check the time
1039                  * here trusting the code is correct.
1040                  */
1041                 return;
1042         }
1043         if (SEQ_GT(tp->snd_max, tp->snd_una) &&
1044             ((tmr_up == PACE_TMR_RXT) ||
1045              (tmr_up == PACE_TMR_TLP) ||
1046              (tmr_up == PACE_TMR_RACK))) {
1047                 /*
1048                  * We have outstanding data and
1049                  * we *do* have a RACK, TLP or RXT
1050                  * timer running. We won't restart
1051                  * anything here since thats probably ok we
1052                  * will get called with some timer here shortly.
1053                  */
1054                 return;
1055         }
1056         /*
1057          * Ok the timer originally started is not what we want now. We will
1058          * force the hpts to be stopped if any, and restart with the slot
1059          * set to what was in the saved slot.
1060          */
1061 wrong_timer:
1062         if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) {
1063                 if (inp->inp_in_hpts)
1064                         tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
1065                 bbr_timer_cancel(bbr, __LINE__, cts);
1066                 bbr_start_hpts_timer(bbr, tp, cts, 1, bbr->r_ctl.rc_last_delay_val,
1067                     0);
1068         } else {
1069                 /*
1070                  * Output is hptsi so we just need to switch the type of
1071                  * timer. We don't bother with keep-alive, since when we
1072                  * jump through the output, it will start the keep-alive if
1073                  * nothing is sent.
1074                  *
1075                  * We only need a delayed-ack added and or the hpts_timeout.
1076                  */
1077                 hpts_timeout = bbr_timer_start(tp, bbr, cts);
1078                 if (tp->t_flags & TF_DELACK) {
1079                         if (hpts_timeout == 0) {
1080                                 hpts_timeout = bbr_delack_time;
1081                                 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1082                         }
1083                         else if (hpts_timeout > bbr_delack_time) {
1084                                 hpts_timeout = bbr_delack_time;
1085                                 bbr->r_ctl.rc_hpts_flags = PACE_TMR_DELACK;
1086                         }
1087                 }
1088                 if (hpts_timeout) {
1089                         if (hpts_timeout > 0x7ffffffe)
1090                                 hpts_timeout = 0x7ffffffe;
1091                         bbr->r_ctl.rc_timer_exp = cts + hpts_timeout;
1092                 }
1093         }
1094 }
1095
1096 int32_t bbr_clear_lost = 0;
1097
1098 /*
1099  * Considers the two time values now (cts) and earlier.
1100  * If cts is smaller than earlier, we could have
1101  * had a sequence wrap (our counter wraps every
1102  * 70 min or so) or it could be just clock skew
1103  * getting us two differnt time values. Clock skew
1104  * will show up within 10ms or so. So in such
1105  * a case (where cts is behind earlier time by
1106  * less than 10ms) we return 0. Otherwise we
1107  * return the true difference between them.
1108  */
1109 static inline uint32_t
1110 bbr_calc_time(uint32_t cts, uint32_t earlier_time) {
1111         /*
1112          * Given two timestamps, the current time stamp cts, and some other
1113          * time-stamp taken in theory earlier return the difference. The
1114          * trick is here sometimes locking will get the other timestamp
1115          * after the cts. If this occurs we need to return 0.
1116          */
1117         if (TSTMP_GEQ(cts, earlier_time))
1118                 return (cts - earlier_time);
1119         /*
1120          * cts is behind earlier_time if its less than 10ms consider it 0.
1121          * If its more than 10ms difference then we had a time wrap. Else
1122          * its just the normal locking foo. I wonder if we should not go to
1123          * 64bit TS and get rid of this issue.
1124          */
1125         if (TSTMP_GEQ((cts + 10000), earlier_time))
1126                 return (0);
1127         /*
1128          * Ok the time must have wrapped. So we need to answer a large
1129          * amount of time, which the normal subtraction should do.
1130          */
1131         return (cts - earlier_time);
1132 }
1133
1134
1135
1136 static int
1137 sysctl_bbr_clear_lost(SYSCTL_HANDLER_ARGS)
1138 {
1139         uint32_t stat;
1140         int32_t error;
1141
1142         error = SYSCTL_OUT(req, &bbr_clear_lost, sizeof(uint32_t));
1143         if (error || req->newptr == NULL)
1144                 return error;
1145
1146         error = SYSCTL_IN(req, &stat, sizeof(uint32_t));
1147         if (error)
1148                 return (error);
1149         if (stat == 1) {
1150 #ifdef BBR_INVARIANTS
1151                 printf("Clearing BBR lost counters\n");
1152 #endif
1153                 COUNTER_ARRAY_ZERO(bbr_state_lost, BBR_MAX_STAT);
1154                 COUNTER_ARRAY_ZERO(bbr_state_time, BBR_MAX_STAT);
1155                 COUNTER_ARRAY_ZERO(bbr_state_resend, BBR_MAX_STAT);
1156         } else if (stat == 2) {
1157 #ifdef BBR_INVARIANTS
1158                 printf("Clearing BBR option counters\n");
1159 #endif
1160                 COUNTER_ARRAY_ZERO(bbr_opts_arry, BBR_OPTS_SIZE);
1161         } else if (stat == 3) {
1162 #ifdef BBR_INVARIANTS
1163                 printf("Clearing BBR stats counters\n");
1164 #endif
1165                 COUNTER_ARRAY_ZERO(bbr_stat_arry, BBR_STAT_SIZE);
1166         } else if (stat == 4) {
1167 #ifdef BBR_INVARIANTS
1168                 printf("Clearing BBR out-size counters\n");
1169 #endif
1170                 COUNTER_ARRAY_ZERO(bbr_out_size, TCP_MSS_ACCT_SIZE);
1171         }
1172         bbr_clear_lost = 0;
1173         return (0);
1174 }
1175
1176 static void
1177 bbr_init_sysctls(void)
1178 {
1179         struct sysctl_oid *bbr_probertt;
1180         struct sysctl_oid *bbr_hptsi;
1181         struct sysctl_oid *bbr_measure;
1182         struct sysctl_oid *bbr_cwnd;
1183         struct sysctl_oid *bbr_timeout;
1184         struct sysctl_oid *bbr_states;
1185         struct sysctl_oid *bbr_startup;
1186         struct sysctl_oid *bbr_policer;
1187
1188         /* Probe rtt controls */
1189         bbr_probertt = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1190             SYSCTL_CHILDREN(bbr_sysctl_root),
1191             OID_AUTO,
1192             "probertt",
1193             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1194             "");
1195         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1196             SYSCTL_CHILDREN(bbr_probertt),
1197             OID_AUTO, "gain", CTLFLAG_RW,
1198             &bbr_rttprobe_gain, 192,
1199             "What is the filter gain drop in probe_rtt (0=disable)?");
1200         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1201             SYSCTL_CHILDREN(bbr_probertt),
1202             OID_AUTO, "cwnd", CTLFLAG_RW,
1203             &bbr_rtt_probe_cwndtarg, 4,
1204             "How many mss's are outstanding during probe-rtt");
1205         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1206             SYSCTL_CHILDREN(bbr_probertt),
1207             OID_AUTO, "int", CTLFLAG_RW,
1208             &bbr_rtt_probe_limit, 4000000,
1209             "If RTT has not shrank in this many micro-seconds enter probe-rtt");
1210         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1211             SYSCTL_CHILDREN(bbr_probertt),
1212             OID_AUTO, "mintime", CTLFLAG_RW,
1213             &bbr_rtt_probe_time, 200000,
1214             "How many microseconds in probe-rtt");
1215         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1216             SYSCTL_CHILDREN(bbr_probertt),
1217             OID_AUTO, "filter_len_sec", CTLFLAG_RW,
1218             &bbr_filter_len_sec, 6,
1219             "How long in seconds does the rttProp filter run?");
1220         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1221             SYSCTL_CHILDREN(bbr_probertt),
1222             OID_AUTO, "drain_rtt", CTLFLAG_RW,
1223             &bbr_drain_rtt, BBR_SRTT,
1224             "What is the drain rtt to use in probeRTT (rtt_prop=0, rtt_rack=1, rtt_pkt=2, rtt_srtt=3?");
1225         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1226             SYSCTL_CHILDREN(bbr_probertt),
1227             OID_AUTO, "can_force", CTLFLAG_RW,
1228             &bbr_can_force_probertt, 0,
1229             "If we keep setting new low rtt's but delay going in probe-rtt can we force in??");
1230         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1231             SYSCTL_CHILDREN(bbr_probertt),
1232             OID_AUTO, "enter_sets_force", CTLFLAG_RW,
1233             &bbr_probertt_sets_rtt, 0,
1234             "In NF mode, do we imitate google_mode and set the rttProp on entry to probe-rtt?");
1235         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1236             SYSCTL_CHILDREN(bbr_probertt),
1237             OID_AUTO, "can_adjust", CTLFLAG_RW,
1238             &bbr_can_adjust_probertt, 1,
1239             "Can we dynamically adjust the probe-rtt limits and times?");
1240         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1241             SYSCTL_CHILDREN(bbr_probertt),
1242             OID_AUTO, "is_ratio", CTLFLAG_RW,
1243             &bbr_is_ratio, 0,
1244             "is the limit to filter a ratio?");
1245         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1246             SYSCTL_CHILDREN(bbr_probertt),
1247             OID_AUTO, "use_cwnd", CTLFLAG_RW,
1248             &bbr_prtt_slam_cwnd, 0,
1249             "Should we set/recover cwnd?");
1250         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1251             SYSCTL_CHILDREN(bbr_probertt),
1252             OID_AUTO, "can_use_ts", CTLFLAG_RW,
1253             &bbr_can_use_ts_for_rtt, 1,
1254             "Can we use the ms timestamp if available for retransmistted rtt calculations?");
1255
1256         /* Pacing controls */
1257         bbr_hptsi = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1258             SYSCTL_CHILDREN(bbr_sysctl_root),
1259             OID_AUTO,
1260             "pacing",
1261             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1262             "");
1263         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1264             SYSCTL_CHILDREN(bbr_hptsi),
1265             OID_AUTO, "hw_pacing", CTLFLAG_RW,
1266             &bbr_allow_hdwr_pacing, 1,
1267             "Do we allow hardware pacing?");
1268         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1269             SYSCTL_CHILDREN(bbr_hptsi),
1270             OID_AUTO, "hw_pacing_limit", CTLFLAG_RW,
1271             &bbr_hardware_pacing_limit, 4000,
1272             "Do we have a limited number of connections for pacing chelsio (0=no limit)?");
1273         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1274             SYSCTL_CHILDREN(bbr_hptsi),
1275             OID_AUTO, "hw_pacing_adj", CTLFLAG_RW,
1276             &bbr_hdwr_pace_adjust, 2,
1277             "Multiplier to calculated tso size?");
1278         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1279             SYSCTL_CHILDREN(bbr_hptsi),
1280             OID_AUTO, "hw_pacing_floor", CTLFLAG_RW,
1281             &bbr_hdwr_pace_floor, 1,
1282             "Do we invoke the hardware pacing floor?");
1283         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1284             SYSCTL_CHILDREN(bbr_hptsi),
1285             OID_AUTO, "hw_pacing_delay_cnt", CTLFLAG_RW,
1286             &bbr_hdwr_pacing_delay_cnt, 10,
1287             "How many packets must be sent after hdwr pacing is enabled");
1288         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1289             SYSCTL_CHILDREN(bbr_hptsi),
1290             OID_AUTO, "bw_cross", CTLFLAG_RW,
1291             &bbr_cross_over, 3000000,
1292             "What is the point where we cross over to linux like TSO size set");
1293         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1294             SYSCTL_CHILDREN(bbr_hptsi),
1295             OID_AUTO, "seg_deltarg", CTLFLAG_RW,
1296             &bbr_hptsi_segments_delay_tar, 7000,
1297             "What is the worse case delay target for hptsi < 48Mbp connections");
1298         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1299             SYSCTL_CHILDREN(bbr_hptsi),
1300             OID_AUTO, "enet_oh", CTLFLAG_RW,
1301             &bbr_include_enet_oh, 0,
1302             "Do we include the ethernet overhead in calculating pacing delay?");
1303         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1304             SYSCTL_CHILDREN(bbr_hptsi),
1305             OID_AUTO, "ip_oh", CTLFLAG_RW,
1306             &bbr_include_ip_oh, 1,
1307             "Do we include the IP overhead in calculating pacing delay?");
1308         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1309             SYSCTL_CHILDREN(bbr_hptsi),
1310             OID_AUTO, "tcp_oh", CTLFLAG_RW,
1311             &bbr_include_tcp_oh, 0,
1312             "Do we include the TCP overhead in calculating pacing delay?");
1313         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1314             SYSCTL_CHILDREN(bbr_hptsi),
1315             OID_AUTO, "google_discount", CTLFLAG_RW,
1316             &bbr_google_discount, 10,
1317             "What is the default google discount percentage wise for pacing (11 = 1.1%%)?");
1318         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1319             SYSCTL_CHILDREN(bbr_hptsi),
1320             OID_AUTO, "all_get_min", CTLFLAG_RW,
1321             &bbr_all_get_min, 0,
1322             "If you are less than a MSS do you just get the min?");
1323         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1324             SYSCTL_CHILDREN(bbr_hptsi),
1325             OID_AUTO, "tso_min", CTLFLAG_RW,
1326             &bbr_hptsi_bytes_min, 1460,
1327             "For 0 -> 24Mbps what is floor number of segments for TSO");
1328         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1329             SYSCTL_CHILDREN(bbr_hptsi),
1330             OID_AUTO, "seg_tso_max", CTLFLAG_RW,
1331             &bbr_hptsi_segments_max, 6,
1332             "For 0 -> 24Mbps what is top number of segments for TSO");
1333         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1334             SYSCTL_CHILDREN(bbr_hptsi),
1335             OID_AUTO, "seg_floor", CTLFLAG_RW,
1336             &bbr_hptsi_segments_floor, 1,
1337             "Minimum TSO size we will fall too in segments");
1338         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1339             SYSCTL_CHILDREN(bbr_hptsi),
1340             OID_AUTO, "utter_max", CTLFLAG_RW,
1341             &bbr_hptsi_utter_max, 0,
1342             "The absolute maximum that any pacing (outside of hardware) can be");
1343         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1344             SYSCTL_CHILDREN(bbr_hptsi),
1345             OID_AUTO, "seg_divisor", CTLFLAG_RW,
1346             &bbr_hptsi_per_second, 100,
1347             "What is the divisor in our hptsi TSO calculation 512Mbps < X > 24Mbps ");
1348         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1349             SYSCTL_CHILDREN(bbr_hptsi),
1350             OID_AUTO, "srtt_mul", CTLFLAG_RW,
1351             &bbr_hptsi_max_mul, 1,
1352             "The multiplier for pace len max");
1353         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1354             SYSCTL_CHILDREN(bbr_hptsi),
1355             OID_AUTO, "srtt_div", CTLFLAG_RW,
1356             &bbr_hptsi_max_div, 2,
1357             "The divisor for pace len max");
1358         /* Measurement controls */
1359         bbr_measure = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1360             SYSCTL_CHILDREN(bbr_sysctl_root),
1361             OID_AUTO,
1362             "measure",
1363             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1364             "Measurement controls");
1365         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1366             SYSCTL_CHILDREN(bbr_measure),
1367             OID_AUTO, "min_i_bw", CTLFLAG_RW,
1368             &bbr_initial_bw_bps, 62500,
1369             "Minimum initial b/w in bytes per second");
1370         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1371             SYSCTL_CHILDREN(bbr_measure),
1372             OID_AUTO, "no_sack_needed", CTLFLAG_RW,
1373             &bbr_sack_not_required, 0,
1374             "Do we allow bbr to run on connections not supporting SACK?");
1375         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1376             SYSCTL_CHILDREN(bbr_measure),
1377             OID_AUTO, "use_google", CTLFLAG_RW,
1378             &bbr_use_google_algo, 0,
1379             "Use has close to google V1.0 has possible?");
1380         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1381             SYSCTL_CHILDREN(bbr_measure),
1382             OID_AUTO, "ts_limiting", CTLFLAG_RW,
1383             &bbr_ts_limiting, 1,
1384             "Do we attempt to use the peers timestamp to limit b/w caculations?");
1385         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1386             SYSCTL_CHILDREN(bbr_measure),
1387             OID_AUTO, "ts_can_raise", CTLFLAG_RW,
1388             &bbr_ts_can_raise, 0,
1389             "Can we raise the b/w via timestamp b/w calculation?");
1390         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1391             SYSCTL_CHILDREN(bbr_measure),
1392             OID_AUTO, "ts_delta", CTLFLAG_RW,
1393             &bbr_min_usec_delta, 20000,
1394             "How long in usec between ts of our sends in ts validation code?");
1395         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1396             SYSCTL_CHILDREN(bbr_measure),
1397             OID_AUTO, "ts_peer_delta", CTLFLAG_RW,
1398             &bbr_min_peer_delta, 20,
1399             "What min numerical value should be between the peer deltas?");
1400         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1401             SYSCTL_CHILDREN(bbr_measure),
1402             OID_AUTO, "ts_delta_percent", CTLFLAG_RW,
1403             &bbr_delta_percent, 150,
1404             "What percentage (150 = 15.0) do we allow variance for?");
1405         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1406             SYSCTL_CHILDREN(bbr_measure),
1407             OID_AUTO, "min_measure_good_bw", CTLFLAG_RW,
1408             &bbr_min_measurements_req, 1,
1409             "What is the minimum measurment count we need before we switch to our b/w estimate");
1410         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1411             SYSCTL_CHILDREN(bbr_measure),
1412             OID_AUTO, "min_measure_before_pace", CTLFLAG_RW,
1413             &bbr_no_pacing_until, 4,
1414             "How many pkt-epoch's (0 is off) do we need before pacing is on?");
1415         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1416             SYSCTL_CHILDREN(bbr_measure),
1417             OID_AUTO, "quanta", CTLFLAG_RW,
1418             &bbr_quanta, 2,
1419             "Extra quanta to add when calculating the target (ID section 4.2.3.2).");
1420         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1421             SYSCTL_CHILDREN(bbr_measure),
1422             OID_AUTO, "noretran", CTLFLAG_RW,
1423             &bbr_no_retran, 0,
1424             "Should google mode not use retransmission measurements for the b/w estimation?");
1425         /* State controls */
1426         bbr_states = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1427             SYSCTL_CHILDREN(bbr_sysctl_root),
1428             OID_AUTO,
1429             "states",
1430             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1431             "State controls");
1432         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1433             SYSCTL_CHILDREN(bbr_states),
1434             OID_AUTO, "idle_restart", CTLFLAG_RW,
1435             &bbr_uses_idle_restart, 0,
1436             "Do we use a new special idle_restart state to ramp back up quickly?");
1437         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1438             SYSCTL_CHILDREN(bbr_states),
1439             OID_AUTO, "idle_restart_threshold", CTLFLAG_RW,
1440             &bbr_idle_restart_threshold, 100000,
1441             "How long must we be idle before we restart??");
1442         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1443             SYSCTL_CHILDREN(bbr_states),
1444             OID_AUTO, "use_pkt_epoch", CTLFLAG_RW,
1445             &bbr_state_is_pkt_epoch, 0,
1446             "Do we use a pkt-epoch for substate if 0 rttProp?");
1447         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1448             SYSCTL_CHILDREN(bbr_states),
1449             OID_AUTO, "startup_rtt_gain", CTLFLAG_RW,
1450             &bbr_rtt_gain_thresh, 0,
1451             "What increase in RTT triggers us to stop ignoring no-loss and possibly exit startup?");
1452         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1453             SYSCTL_CHILDREN(bbr_states),
1454             OID_AUTO, "drain_floor", CTLFLAG_RW,
1455             &bbr_drain_floor, 88,
1456             "What is the lowest we can drain (pg) too?");
1457         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1458             SYSCTL_CHILDREN(bbr_states),
1459             OID_AUTO, "drain_2_target", CTLFLAG_RW,
1460             &bbr_state_drain_2_tar, 1,
1461             "Do we drain to target in drain substate?");
1462         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1463             SYSCTL_CHILDREN(bbr_states),
1464             OID_AUTO, "gain_2_target", CTLFLAG_RW,
1465             &bbr_gain_to_target, 1,
1466             "Does probe bw gain to target??");
1467         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1468             SYSCTL_CHILDREN(bbr_states),
1469             OID_AUTO, "gain_extra_time", CTLFLAG_RW,
1470             &bbr_gain_gets_extra_too, 1,
1471             "Does probe bw gain get the extra time too?");
1472         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1473             SYSCTL_CHILDREN(bbr_states),
1474             OID_AUTO, "ld_div", CTLFLAG_RW,
1475             &bbr_drain_drop_div, 5,
1476             "Long drain drop divider?");
1477         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1478             SYSCTL_CHILDREN(bbr_states),
1479             OID_AUTO, "ld_mul", CTLFLAG_RW,
1480             &bbr_drain_drop_mul, 4,
1481             "Long drain drop multiplier?");
1482         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1483             SYSCTL_CHILDREN(bbr_states),
1484             OID_AUTO, "rand_ot_disc", CTLFLAG_RW,
1485             &bbr_rand_ot, 50,
1486             "Random discount of the ot?");
1487         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1488             SYSCTL_CHILDREN(bbr_states),
1489             OID_AUTO, "dr_filter_life", CTLFLAG_RW,
1490             &bbr_num_pktepo_for_del_limit, BBR_NUM_RTTS_FOR_DEL_LIMIT,
1491             "How many packet-epochs does the b/w delivery rate last?");
1492         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1493             SYSCTL_CHILDREN(bbr_states),
1494             OID_AUTO, "subdrain_applimited", CTLFLAG_RW,
1495             &bbr_sub_drain_app_limit, 0,
1496             "Does our sub-state drain invoke app limited if its long?");
1497         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1498             SYSCTL_CHILDREN(bbr_states),
1499             OID_AUTO, "use_cwnd_subdrain", CTLFLAG_RW,
1500             &bbr_sub_drain_slam_cwnd, 0,
1501             "Should we set/recover cwnd for sub-state drain?");
1502         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1503             SYSCTL_CHILDREN(bbr_states),
1504             OID_AUTO, "use_cwnd_maindrain", CTLFLAG_RW,
1505             &bbr_slam_cwnd_in_main_drain, 0,
1506             "Should we set/recover cwnd for main-state drain?");
1507         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1508             SYSCTL_CHILDREN(bbr_states),
1509             OID_AUTO, "google_gets_earlyout", CTLFLAG_RW,
1510             &google_allow_early_out, 1,
1511             "Should we allow google probe-bw/drain to exit early at flight target?");
1512         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1513             SYSCTL_CHILDREN(bbr_states),
1514             OID_AUTO, "google_exit_loss", CTLFLAG_RW,
1515             &google_consider_lost, 1,
1516             "Should we have losses exit gain of probebw in google mode??");
1517         /* Startup controls */
1518         bbr_startup = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1519             SYSCTL_CHILDREN(bbr_sysctl_root),
1520             OID_AUTO,
1521             "startup",
1522             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1523             "Startup controls");
1524         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1525             SYSCTL_CHILDREN(bbr_startup),
1526             OID_AUTO, "cheat_iwnd", CTLFLAG_RW,
1527             &bbr_sends_full_iwnd, 1,
1528             "Do we not pace but burst out initial windows has our TSO size?");
1529         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1530             SYSCTL_CHILDREN(bbr_startup),
1531             OID_AUTO, "loss_threshold", CTLFLAG_RW,
1532             &bbr_startup_loss_thresh, 2000,
1533             "In startup what is the loss threshold in a pe that will exit us from startup?");
1534         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1535             SYSCTL_CHILDREN(bbr_startup),
1536             OID_AUTO, "use_lowerpg", CTLFLAG_RW,
1537             &bbr_use_lower_gain_in_startup, 1,
1538             "Should we use a lower hptsi gain if we see loss in startup?");
1539         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1540             SYSCTL_CHILDREN(bbr_startup),
1541             OID_AUTO, "gain", CTLFLAG_RW,
1542             &bbr_start_exit, 25,
1543             "What gain percent do we need to see to stay in startup??");
1544         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1545             SYSCTL_CHILDREN(bbr_startup),
1546             OID_AUTO, "low_gain", CTLFLAG_RW,
1547             &bbr_low_start_exit, 15,
1548             "What gain percent do we need to see to stay in the lower gain startup??");
1549         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1550             SYSCTL_CHILDREN(bbr_startup),
1551             OID_AUTO, "loss_exit", CTLFLAG_RW,
1552             &bbr_exit_startup_at_loss, 1,
1553             "Should we exit startup at loss in an epoch if we are not gaining?");
1554         /* CWND controls */
1555         bbr_cwnd = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1556             SYSCTL_CHILDREN(bbr_sysctl_root),
1557             OID_AUTO,
1558             "cwnd",
1559             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1560             "Cwnd controls");
1561         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1562             SYSCTL_CHILDREN(bbr_cwnd),
1563             OID_AUTO, "tar_rtt", CTLFLAG_RW,
1564             &bbr_cwndtarget_rtt_touse, 0,
1565             "Target cwnd rtt measurment to use (0=rtt_prop, 1=rtt_rack, 2=pkt_rtt, 3=srtt)?");
1566         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1567             SYSCTL_CHILDREN(bbr_cwnd),
1568             OID_AUTO, "may_shrink", CTLFLAG_RW,
1569             &bbr_cwnd_may_shrink, 0,
1570             "Can the cwnd shrink if it would grow to more than the target?");
1571         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1572             SYSCTL_CHILDREN(bbr_cwnd),
1573             OID_AUTO, "max_target_limit", CTLFLAG_RW,
1574             &bbr_target_cwnd_mult_limit, 8,
1575             "Do we limit the cwnd to some multiple of the cwnd target if cwnd can't shrink 0=no?");
1576         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1577             SYSCTL_CHILDREN(bbr_cwnd),
1578             OID_AUTO, "highspeed_min", CTLFLAG_RW,
1579             &bbr_cwnd_min_val_hs, BBR_HIGHSPEED_NUM_MSS,
1580             "What is the high-speed min cwnd (rttProp under 1ms)");
1581         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1582             SYSCTL_CHILDREN(bbr_cwnd),
1583             OID_AUTO, "lowspeed_min", CTLFLAG_RW,
1584             &bbr_cwnd_min_val, BBR_PROBERTT_NUM_MSS,
1585             "What is the min cwnd (rttProp > 1ms)");
1586         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1587             SYSCTL_CHILDREN(bbr_cwnd),
1588             OID_AUTO, "initwin", CTLFLAG_RW,
1589             &bbr_def_init_win, 10,
1590             "What is the BBR initial window, if 0 use tcp version");
1591         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1592             SYSCTL_CHILDREN(bbr_cwnd),
1593             OID_AUTO, "do_loss_red", CTLFLAG_RW,
1594             &bbr_do_red, 600,
1595             "Do we reduce the b/w at exit from recovery based on ratio of prop/srtt (800=80.0, 0=off)?");
1596         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1597             SYSCTL_CHILDREN(bbr_cwnd),
1598             OID_AUTO, "red_scale", CTLFLAG_RW,
1599             &bbr_red_scale, 20000,
1600             "What RTT do we scale with?");
1601         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1602             SYSCTL_CHILDREN(bbr_cwnd),
1603             OID_AUTO, "red_growslow", CTLFLAG_RW,
1604             &bbr_red_growth_restrict, 1,
1605             "Do we restrict cwnd growth for whats in flight?");
1606         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1607             SYSCTL_CHILDREN(bbr_cwnd),
1608             OID_AUTO, "red_div", CTLFLAG_RW,
1609             &bbr_red_div, 2,
1610             "If we reduce whats the divisor?");
1611         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1612             SYSCTL_CHILDREN(bbr_cwnd),
1613             OID_AUTO, "red_mul", CTLFLAG_RW,
1614             &bbr_red_mul, 1,
1615             "If we reduce whats the mulitiplier?");
1616         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1617             SYSCTL_CHILDREN(bbr_cwnd),
1618             OID_AUTO, "target_is_unit", CTLFLAG_RW,
1619             &bbr_target_is_bbunit, 0,
1620             "Is the state target the pacing_gain or BBR_UNIT?");
1621         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1622             SYSCTL_CHILDREN(bbr_cwnd),
1623             OID_AUTO, "drop_limit", CTLFLAG_RW,
1624             &bbr_drop_limit, 0,
1625             "Number of segments limit for drop (0=use min_cwnd w/flight)?");
1626
1627         /* Timeout controls */
1628         bbr_timeout = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1629             SYSCTL_CHILDREN(bbr_sysctl_root),
1630             OID_AUTO,
1631             "timeout",
1632             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1633             "Time out controls");
1634         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1635             SYSCTL_CHILDREN(bbr_timeout),
1636             OID_AUTO, "delack", CTLFLAG_RW,
1637             &bbr_delack_time, 100000,
1638             "BBR's delayed ack time");
1639         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1640             SYSCTL_CHILDREN(bbr_timeout),
1641             OID_AUTO, "tlp_uses", CTLFLAG_RW,
1642             &bbr_tlp_type_to_use, 3,
1643             "RTT that TLP uses in its calculations, 0=rttProp, 1=Rack_rtt, 2=pkt_rtt and 3=srtt");
1644         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1645             SYSCTL_CHILDREN(bbr_timeout),
1646             OID_AUTO, "persmin", CTLFLAG_RW,
1647             &bbr_persist_min, 250000,
1648             "What is the minimum time in microseconds between persists");
1649         SYSCTL_ADD_U32(&bbr_sysctl_ctx,
1650             SYSCTL_CHILDREN(bbr_timeout),
1651             OID_AUTO, "persmax", CTLFLAG_RW,
1652             &bbr_persist_max, 1000000,
1653             "What is the largest delay in microseconds between persists");
1654         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1655             SYSCTL_CHILDREN(bbr_timeout),
1656             OID_AUTO, "tlp_minto", CTLFLAG_RW,
1657             &bbr_tlp_min, 10000,
1658             "TLP Min timeout in usecs");
1659         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1660             SYSCTL_CHILDREN(bbr_timeout),
1661             OID_AUTO, "tlp_dack_time", CTLFLAG_RW,
1662             &bbr_delayed_ack_time, 200000,
1663             "TLP delayed ack compensation value");
1664         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1665             SYSCTL_CHILDREN(bbr_sysctl_root),
1666             OID_AUTO, "minrto", CTLFLAG_RW,
1667             &bbr_rto_min_ms, 30,
1668             "Minimum RTO in ms");
1669         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1670             SYSCTL_CHILDREN(bbr_timeout),
1671             OID_AUTO, "maxrto", CTLFLAG_RW,
1672             &bbr_rto_max_sec, 4,
1673             "Maxiumum RTO in seconds -- should be at least as large as min_rto");
1674         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1675             SYSCTL_CHILDREN(bbr_timeout),
1676             OID_AUTO, "tlp_retry", CTLFLAG_RW,
1677             &bbr_tlp_max_resend, 2,
1678             "How many times does TLP retry a single segment or multiple with no ACK");
1679         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1680             SYSCTL_CHILDREN(bbr_timeout),
1681             OID_AUTO, "minto", CTLFLAG_RW,
1682             &bbr_min_to, 1000,
1683             "Minimum rack timeout in useconds");
1684         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1685             SYSCTL_CHILDREN(bbr_timeout),
1686             OID_AUTO, "pktdelay", CTLFLAG_RW,
1687             &bbr_pkt_delay, 1000,
1688             "Extra RACK time (in useconds) besides reordering thresh");
1689         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1690             SYSCTL_CHILDREN(bbr_timeout),
1691             OID_AUTO, "incr_tmrs", CTLFLAG_RW,
1692             &bbr_incr_timers, 1,
1693             "Increase the RXT/TLP timer by the pacing time used?");
1694         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1695             SYSCTL_CHILDREN(bbr_timeout),
1696             OID_AUTO, "rxtmark_sackpassed", CTLFLAG_RW,
1697             &bbr_marks_rxt_sack_passed, 0,
1698             "Mark sack passed on all those not ack'd when a RXT hits?");
1699         /* Policer controls */
1700         bbr_policer = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
1701             SYSCTL_CHILDREN(bbr_sysctl_root),
1702             OID_AUTO,
1703             "policer",
1704             CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1705             "Policer controls");
1706         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1707             SYSCTL_CHILDREN(bbr_policer),
1708             OID_AUTO, "detect_enable", CTLFLAG_RW,
1709             &bbr_policer_detection_enabled, 1,
1710             "Is policer detection enabled??");
1711         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1712             SYSCTL_CHILDREN(bbr_policer),
1713             OID_AUTO, "min_pes", CTLFLAG_RW,
1714             &bbr_lt_intvl_min_rtts, 4,
1715             "Minimum number of PE's?");
1716         SYSCTL_ADD_U64(&bbr_sysctl_ctx,
1717             SYSCTL_CHILDREN(bbr_policer),
1718             OID_AUTO, "bwdiff", CTLFLAG_RW,
1719             &bbr_lt_bw_diff, (4000/8),
1720             "Minimal bw diff?");
1721         SYSCTL_ADD_U64(&bbr_sysctl_ctx,
1722             SYSCTL_CHILDREN(bbr_policer),
1723             OID_AUTO, "bwratio", CTLFLAG_RW,
1724             &bbr_lt_bw_ratio, 8,
1725             "Minimal bw diff?");
1726         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1727             SYSCTL_CHILDREN(bbr_policer),
1728             OID_AUTO, "from_rack_rxt", CTLFLAG_RW,
1729             &bbr_policer_call_from_rack_to, 0,
1730             "Do we call the policer detection code from a rack-timeout?");
1731         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1732             SYSCTL_CHILDREN(bbr_policer),
1733             OID_AUTO, "false_postive", CTLFLAG_RW,
1734             &bbr_lt_intvl_fp, 0,
1735             "What packet epoch do we do false-postive detection at (0=no)?");
1736         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1737             SYSCTL_CHILDREN(bbr_policer),
1738             OID_AUTO, "loss_thresh", CTLFLAG_RW,
1739             &bbr_lt_loss_thresh, 196,
1740             "Loss threshold 196 = 19.6%?");
1741         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1742             SYSCTL_CHILDREN(bbr_policer),
1743             OID_AUTO, "false_postive_thresh", CTLFLAG_RW,
1744             &bbr_lt_fd_thresh, 100,
1745             "What percentage is the false detection threshold (150=15.0)?");
1746         /* All the rest */
1747         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1748             SYSCTL_CHILDREN(bbr_sysctl_root),
1749             OID_AUTO, "cheat_rxt", CTLFLAG_RW,
1750             &bbr_use_rack_resend_cheat, 0,
1751             "Do we burst 1ms between sends on retransmissions (like rack)?");
1752         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1753             SYSCTL_CHILDREN(bbr_sysctl_root),
1754             OID_AUTO, "error_paceout", CTLFLAG_RW,
1755             &bbr_error_base_paceout, 10000,
1756             "When we hit an error what is the min to pace out in usec's?");
1757         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1758             SYSCTL_CHILDREN(bbr_sysctl_root),
1759             OID_AUTO, "kill_paceout", CTLFLAG_RW,
1760             &bbr_max_net_error_cnt, 10,
1761             "When we hit this many errors in a row, kill the session?");
1762         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1763             SYSCTL_CHILDREN(bbr_sysctl_root),
1764             OID_AUTO, "data_after_close", CTLFLAG_RW,
1765             &bbr_ignore_data_after_close, 1,
1766             "Do we hold off sending a RST until all pending data is ack'd");
1767         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1768             SYSCTL_CHILDREN(bbr_sysctl_root),
1769             OID_AUTO, "resend_use_tso", CTLFLAG_RW,
1770             &bbr_resends_use_tso, 0,
1771             "Can resends use TSO?");
1772         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1773             SYSCTL_CHILDREN(bbr_sysctl_root),
1774             OID_AUTO, "sblklimit", CTLFLAG_RW,
1775             &bbr_sack_block_limit, 128,
1776             "When do we start ignoring small sack blocks");
1777         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1778             SYSCTL_CHILDREN(bbr_sysctl_root),
1779             OID_AUTO, "bb_verbose", CTLFLAG_RW,
1780             &bbr_verbose_logging, 0,
1781             "Should BBR black box logging be verbose");
1782         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1783             SYSCTL_CHILDREN(bbr_sysctl_root),
1784             OID_AUTO, "reorder_thresh", CTLFLAG_RW,
1785             &bbr_reorder_thresh, 2,
1786             "What factor for rack will be added when seeing reordering (shift right)");
1787         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1788             SYSCTL_CHILDREN(bbr_sysctl_root),
1789             OID_AUTO, "reorder_fade", CTLFLAG_RW,
1790             &bbr_reorder_fade, 0,
1791             "Does reorder detection fade, if so how many ms (0 means never)");
1792         SYSCTL_ADD_S32(&bbr_sysctl_ctx,
1793             SYSCTL_CHILDREN(bbr_sysctl_root),
1794             OID_AUTO, "rtt_tlp_thresh", CTLFLAG_RW,
1795             &bbr_tlp_thresh, 1,
1796             "what divisor for TLP rtt/retran will be added (1=rtt, 2=1/2 rtt etc)");
1797         /* Stats and counters */
1798         /* The pacing counters for hdwr/software can't be in the array */
1799         bbr_nohdwr_pacing_enobuf = counter_u64_alloc(M_WAITOK);
1800         bbr_hdwr_pacing_enobuf = counter_u64_alloc(M_WAITOK);
1801         SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1802             SYSCTL_CHILDREN(bbr_sysctl_root),
1803             OID_AUTO, "enob_hdwr_pacing", CTLFLAG_RD,
1804             &bbr_hdwr_pacing_enobuf,
1805             "Total number of enobufs for hardware paced flows");
1806         SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1807             SYSCTL_CHILDREN(bbr_sysctl_root),
1808             OID_AUTO, "enob_no_hdwr_pacing", CTLFLAG_RD,
1809             &bbr_nohdwr_pacing_enobuf,
1810             "Total number of enobufs for non-hardware paced flows");
1811
1812
1813         bbr_flows_whdwr_pacing = counter_u64_alloc(M_WAITOK);
1814         SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1815             SYSCTL_CHILDREN(bbr_sysctl_root),
1816             OID_AUTO, "hdwr_pacing", CTLFLAG_RD,
1817             &bbr_flows_whdwr_pacing,
1818             "Total number of hardware paced flows");
1819         bbr_flows_nohdwr_pacing = counter_u64_alloc(M_WAITOK);
1820         SYSCTL_ADD_COUNTER_U64(&bbr_sysctl_ctx,
1821             SYSCTL_CHILDREN(bbr_sysctl_root),
1822             OID_AUTO, "software_pacing", CTLFLAG_RD,
1823             &bbr_flows_nohdwr_pacing,
1824             "Total number of software paced flows");
1825         COUNTER_ARRAY_ALLOC(bbr_stat_arry, BBR_STAT_SIZE, M_WAITOK);
1826         SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1827             OID_AUTO, "stats", CTLFLAG_RD,
1828             bbr_stat_arry, BBR_STAT_SIZE, "BBR Stats");
1829         COUNTER_ARRAY_ALLOC(bbr_opts_arry, BBR_OPTS_SIZE, M_WAITOK);
1830         SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1831             OID_AUTO, "opts", CTLFLAG_RD,
1832             bbr_opts_arry, BBR_OPTS_SIZE, "BBR Option Stats");
1833         COUNTER_ARRAY_ALLOC(bbr_state_lost, BBR_MAX_STAT, M_WAITOK);
1834         SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1835             OID_AUTO, "lost", CTLFLAG_RD,
1836             bbr_state_lost, BBR_MAX_STAT, "Stats of when losses occur");
1837         COUNTER_ARRAY_ALLOC(bbr_state_resend, BBR_MAX_STAT, M_WAITOK);
1838         SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1839             OID_AUTO, "stateresend", CTLFLAG_RD,
1840             bbr_state_resend, BBR_MAX_STAT, "Stats of what states resend");
1841         COUNTER_ARRAY_ALLOC(bbr_state_time, BBR_MAX_STAT, M_WAITOK);
1842         SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1843             OID_AUTO, "statetime", CTLFLAG_RD,
1844             bbr_state_time, BBR_MAX_STAT, "Stats of time spent in the states");
1845         COUNTER_ARRAY_ALLOC(bbr_out_size, TCP_MSS_ACCT_SIZE, M_WAITOK);
1846         SYSCTL_ADD_COUNTER_U64_ARRAY(&bbr_sysctl_ctx, SYSCTL_CHILDREN(bbr_sysctl_root),
1847             OID_AUTO, "outsize", CTLFLAG_RD,
1848             bbr_out_size, TCP_MSS_ACCT_SIZE, "Size of output calls");
1849         SYSCTL_ADD_PROC(&bbr_sysctl_ctx,
1850             SYSCTL_CHILDREN(bbr_sysctl_root),
1851             OID_AUTO, "clrlost", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
1852             &bbr_clear_lost, 0, sysctl_bbr_clear_lost, "IU", "Clear lost counters");
1853 }
1854
1855 static inline int32_t
1856 bbr_progress_timeout_check(struct tcp_bbr *bbr)
1857 {
1858         if (bbr->rc_tp->t_maxunacktime && bbr->rc_tp->t_acktime &&
1859             TSTMP_GT(ticks, bbr->rc_tp->t_acktime)) {
1860                 if ((((uint32_t)ticks - bbr->rc_tp->t_acktime)) >= bbr->rc_tp->t_maxunacktime) {
1861                         /*
1862                          * There is an assumption here that the caller will
1863                          * drop the connection, so we increment the
1864                          * statistics.
1865                          */
1866                         bbr_log_progress_event(bbr, bbr->rc_tp, ticks, PROGRESS_DROP, __LINE__);
1867                         BBR_STAT_INC(bbr_progress_drops);
1868 #ifdef NETFLIX_STATS
1869                         KMOD_TCPSTAT_INC(tcps_progdrops);
1870 #endif
1871                         return (1);
1872                 }
1873         }
1874         return (0);
1875 }
1876
1877 static void
1878 bbr_counter_destroy(void)
1879 {
1880         COUNTER_ARRAY_FREE(bbr_stat_arry, BBR_STAT_SIZE);
1881         COUNTER_ARRAY_FREE(bbr_opts_arry, BBR_OPTS_SIZE);
1882         COUNTER_ARRAY_FREE(bbr_out_size, TCP_MSS_ACCT_SIZE);
1883         COUNTER_ARRAY_FREE(bbr_state_lost, BBR_MAX_STAT);
1884         COUNTER_ARRAY_FREE(bbr_state_time, BBR_MAX_STAT);
1885         COUNTER_ARRAY_FREE(bbr_state_resend, BBR_MAX_STAT);
1886         counter_u64_free(bbr_flows_whdwr_pacing);
1887         counter_u64_free(bbr_flows_nohdwr_pacing);
1888
1889 }
1890
1891 static __inline void
1892 bbr_fill_in_logging_data(struct tcp_bbr *bbr, struct tcp_log_bbr *l, uint32_t cts)
1893 {
1894         memset(l, 0, sizeof(union tcp_log_stackspecific));
1895         l->cur_del_rate = bbr->r_ctl.rc_bbr_cur_del_rate;
1896         l->delRate = get_filter_value(&bbr->r_ctl.rc_delrate);
1897         l->rttProp = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
1898         l->bw_inuse = bbr_get_bw(bbr);
1899         l->inflight = ctf_flight_size(bbr->rc_tp,
1900                           (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
1901         l->applimited = bbr->r_ctl.r_app_limited_until;
1902         l->delivered = bbr->r_ctl.rc_delivered;
1903         l->timeStamp = cts;
1904         l->lost = bbr->r_ctl.rc_lost;
1905         l->bbr_state = bbr->rc_bbr_state;
1906         l->bbr_substate = bbr_state_val(bbr);
1907         l->epoch = bbr->r_ctl.rc_rtt_epoch;
1908         l->lt_epoch = bbr->r_ctl.rc_lt_epoch;
1909         l->pacing_gain = bbr->r_ctl.rc_bbr_hptsi_gain;
1910         l->cwnd_gain = bbr->r_ctl.rc_bbr_cwnd_gain;
1911         l->inhpts = bbr->rc_inp->inp_in_hpts;
1912         l->ininput = bbr->rc_inp->inp_in_input;
1913         l->use_lt_bw = bbr->rc_lt_use_bw;
1914         l->pkts_out = bbr->r_ctl.rc_flight_at_input;
1915         l->pkt_epoch = bbr->r_ctl.rc_pkt_epoch;
1916 }
1917
1918 static void
1919 bbr_log_type_bw_reduce(struct tcp_bbr *bbr, int reason)
1920 {
1921         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1922                 union tcp_log_stackspecific log;
1923
1924                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1925                 log.u_bbr.flex1 = 0;
1926                 log.u_bbr.flex2 = 0;
1927                 log.u_bbr.flex5 = 0;
1928                 log.u_bbr.flex3 = 0;
1929                 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_loss_rate;
1930                 log.u_bbr.flex7 = reason;
1931                 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_enters_probertt;
1932                 log.u_bbr.flex8 = 0;
1933                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1934                     &bbr->rc_inp->inp_socket->so_rcv,
1935                     &bbr->rc_inp->inp_socket->so_snd,
1936                     BBR_LOG_BW_RED_EV, 0,
1937                     0, &log, false, &bbr->rc_tv);
1938         }
1939 }
1940
1941 static void
1942 bbr_log_type_rwnd_collapse(struct tcp_bbr *bbr, int seq, int mode, uint32_t count)
1943 {
1944         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1945                 union tcp_log_stackspecific log;
1946
1947                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1948                 log.u_bbr.flex1 = seq;
1949                 log.u_bbr.flex2 = count;
1950                 log.u_bbr.flex8 = mode;
1951                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1952                     &bbr->rc_inp->inp_socket->so_rcv,
1953                     &bbr->rc_inp->inp_socket->so_snd,
1954                     BBR_LOG_LOWGAIN, 0,
1955                     0, &log, false, &bbr->rc_tv);
1956         }
1957 }
1958
1959
1960
1961 static void
1962 bbr_log_type_just_return(struct tcp_bbr *bbr, uint32_t cts, uint32_t tlen, uint8_t hpts_calling,
1963     uint8_t reason, uint32_t p_maxseg, int len)
1964 {
1965         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1966                 union tcp_log_stackspecific log;
1967
1968                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
1969                 log.u_bbr.flex1 = p_maxseg;
1970                 log.u_bbr.flex2 = bbr->r_ctl.rc_hpts_flags;
1971                 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
1972                 log.u_bbr.flex4 = reason;
1973                 log.u_bbr.flex5 = bbr->rc_in_persist;
1974                 log.u_bbr.flex6 = bbr->r_ctl.rc_last_delay_val;
1975                 log.u_bbr.flex7 = p_maxseg;
1976                 log.u_bbr.flex8 = bbr->rc_in_persist;
1977                 log.u_bbr.pkts_out = 0;
1978                 log.u_bbr.applimited = len;
1979                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1980                     &bbr->rc_inp->inp_socket->so_rcv,
1981                     &bbr->rc_inp->inp_socket->so_snd,
1982                     BBR_LOG_JUSTRET, 0,
1983                     tlen, &log, false, &bbr->rc_tv);
1984         }
1985 }
1986
1987
1988 static void
1989 bbr_log_type_enter_rec(struct tcp_bbr *bbr, uint32_t seq)
1990 {
1991         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
1992                 union tcp_log_stackspecific log;
1993
1994                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
1995                 log.u_bbr.flex1 = seq;
1996                 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
1997                 log.u_bbr.flex3 = bbr->r_ctl.rc_recovery_start;
1998                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
1999                     &bbr->rc_inp->inp_socket->so_rcv,
2000                     &bbr->rc_inp->inp_socket->so_snd,
2001                     BBR_LOG_ENTREC, 0,
2002                     0, &log, false, &bbr->rc_tv);
2003         }
2004 }
2005
2006 static void
2007 bbr_log_msgsize_fail(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t len, uint32_t maxseg, uint32_t mtu, int32_t csum_flags, int32_t tso, uint32_t cts)
2008 {
2009         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
2010                 union tcp_log_stackspecific log;
2011
2012                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2013                 log.u_bbr.flex1 = tso;
2014                 log.u_bbr.flex2 = maxseg;
2015                 log.u_bbr.flex3 = mtu;
2016                 log.u_bbr.flex4 = csum_flags;
2017                 TCP_LOG_EVENTP(tp, NULL,
2018                     &bbr->rc_inp->inp_socket->so_rcv,
2019                     &bbr->rc_inp->inp_socket->so_snd,
2020                     BBR_LOG_MSGSIZE, 0,
2021                     0, &log, false, &bbr->rc_tv);
2022         }
2023 }
2024
2025 static void
2026 bbr_log_flowend(struct tcp_bbr *bbr)
2027 {
2028         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2029                 union tcp_log_stackspecific log;
2030                 struct sockbuf *r, *s;
2031                 struct timeval tv;
2032
2033                 if (bbr->rc_inp->inp_socket) {
2034                         r = &bbr->rc_inp->inp_socket->so_rcv;
2035                         s = &bbr->rc_inp->inp_socket->so_snd;
2036                 } else {
2037                         r = s = NULL;
2038                 }
2039                 bbr_fill_in_logging_data(bbr, &log.u_bbr, tcp_get_usecs(&tv));
2040                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2041                     r, s,
2042                     TCP_LOG_FLOWEND, 0,
2043                     0, &log, false, &tv);
2044         }
2045 }
2046
2047 static void
2048 bbr_log_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line,
2049     uint32_t lost, uint32_t del)
2050 {
2051         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2052                 union tcp_log_stackspecific log;
2053
2054                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2055                 log.u_bbr.flex1 = lost;
2056                 log.u_bbr.flex2 = del;
2057                 log.u_bbr.flex3 = bbr->r_ctl.rc_bbr_lastbtlbw;
2058                 log.u_bbr.flex4 = bbr->r_ctl.rc_pkt_epoch_rtt;
2059                 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2060                 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2061                 log.u_bbr.flex7 = line;
2062                 log.u_bbr.flex8 = 0;
2063                 log.u_bbr.inflight = bbr->r_ctl.r_measurement_count;
2064                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2065                     &bbr->rc_inp->inp_socket->so_rcv,
2066                     &bbr->rc_inp->inp_socket->so_snd,
2067                     BBR_LOG_PKT_EPOCH, 0,
2068                     0, &log, false, &bbr->rc_tv);
2069         }
2070 }
2071
2072 static void
2073 bbr_log_time_epoch(struct tcp_bbr *bbr, uint32_t cts, uint32_t line, uint32_t epoch_time)
2074 {
2075         if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2076                 union tcp_log_stackspecific log;
2077
2078                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2079                 log.u_bbr.flex1 = bbr->r_ctl.rc_lost;
2080                 log.u_bbr.flex2 = bbr->rc_inp->inp_socket->so_snd.sb_lowat;
2081                 log.u_bbr.flex3 = bbr->rc_inp->inp_socket->so_snd.sb_hiwat;
2082                 log.u_bbr.flex7 = line;
2083                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2084                     &bbr->rc_inp->inp_socket->so_rcv,
2085                     &bbr->rc_inp->inp_socket->so_snd,
2086                     BBR_LOG_TIME_EPOCH, 0,
2087                     0, &log, false, &bbr->rc_tv);
2088         }
2089 }
2090
2091 static void
2092 bbr_log_set_of_state_target(struct tcp_bbr *bbr, uint32_t new_tar, int line, int meth)
2093 {
2094         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2095                 union tcp_log_stackspecific log;
2096
2097                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2098                 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2099                 log.u_bbr.flex2 = new_tar;
2100                 log.u_bbr.flex3 = line;
2101                 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2102                 log.u_bbr.flex5 = bbr_quanta;
2103                 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_min_segs;
2104                 log.u_bbr.flex7 = bbr->rc_last_options;
2105                 log.u_bbr.flex8 = meth;
2106                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2107                     &bbr->rc_inp->inp_socket->so_rcv,
2108                     &bbr->rc_inp->inp_socket->so_snd,
2109                     BBR_LOG_STATE_TARGET, 0,
2110                     0, &log, false, &bbr->rc_tv);
2111         }
2112
2113 }
2114
2115 static void
2116 bbr_log_type_statechange(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2117 {
2118         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2119                 union tcp_log_stackspecific log;
2120
2121                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2122                 log.u_bbr.flex1 = line;
2123                 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2124                 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2125                 if (bbr_state_is_pkt_epoch)
2126                         log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PKTRTT);
2127                 else
2128                         log.u_bbr.flex4 = bbr_get_rtt(bbr, BBR_RTT_PROP);
2129                 log.u_bbr.flex5 = bbr->r_ctl.rc_bbr_last_startup_epoch;
2130                 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2131                 log.u_bbr.flex7 = (bbr->r_ctl.rc_target_at_state/1000);
2132                 log.u_bbr.lt_epoch = bbr->r_ctl.rc_level_state_extra;
2133                 log.u_bbr.pkts_out = bbr->r_ctl.rc_target_at_state;
2134                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2135                     &bbr->rc_inp->inp_socket->so_rcv,
2136                     &bbr->rc_inp->inp_socket->so_snd,
2137                     BBR_LOG_STATE, 0,
2138                     0, &log, false, &bbr->rc_tv);
2139         }
2140 }
2141
2142 static void
2143 bbr_log_rtt_shrinks(struct tcp_bbr *bbr, uint32_t cts, uint32_t applied,
2144                     uint32_t rtt, uint32_t line, uint8_t reas, uint16_t cond)
2145 {
2146         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2147                 union tcp_log_stackspecific log;
2148
2149                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2150                 log.u_bbr.flex1 = line;
2151                 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2152                 log.u_bbr.flex3 = bbr->r_ctl.last_in_probertt;
2153                 log.u_bbr.flex4 = applied;
2154                 log.u_bbr.flex5 = rtt;
2155                 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2156                 log.u_bbr.flex7 = cond;
2157                 log.u_bbr.flex8 = reas;
2158                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2159                     &bbr->rc_inp->inp_socket->so_rcv,
2160                     &bbr->rc_inp->inp_socket->so_snd,
2161                     BBR_LOG_RTT_SHRINKS, 0,
2162                     0, &log, false, &bbr->rc_tv);
2163         }
2164 }
2165
2166 static void
2167 bbr_log_type_exit_rec(struct tcp_bbr *bbr)
2168 {
2169         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2170                 union tcp_log_stackspecific log;
2171
2172                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2173                 log.u_bbr.flex1 = bbr->r_ctl.rc_recovery_start;
2174                 log.u_bbr.flex2 = bbr->r_ctl.rc_cwnd_on_ent;
2175                 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2176                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2177                     &bbr->rc_inp->inp_socket->so_rcv,
2178                     &bbr->rc_inp->inp_socket->so_snd,
2179                     BBR_LOG_EXITREC, 0,
2180                     0, &log, false, &bbr->rc_tv);
2181         }
2182 }
2183
2184 static void
2185 bbr_log_type_cwndupd(struct tcp_bbr *bbr, uint32_t bytes_this_ack, uint32_t chg,
2186     uint32_t prev_acked, int32_t meth, uint32_t target, uint32_t th_ack, int32_t line)
2187 {
2188         if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2189                 union tcp_log_stackspecific log;
2190
2191                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2192                 log.u_bbr.flex1 = line;
2193                 log.u_bbr.flex2 = prev_acked;
2194                 log.u_bbr.flex3 = bytes_this_ack;
2195                 log.u_bbr.flex4 = chg;
2196                 log.u_bbr.flex5 = th_ack;
2197                 log.u_bbr.flex6 = target;
2198                 log.u_bbr.flex8 = meth;
2199                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2200                     &bbr->rc_inp->inp_socket->so_rcv,
2201                     &bbr->rc_inp->inp_socket->so_snd,
2202                     BBR_LOG_CWND, 0,
2203                     0, &log, false, &bbr->rc_tv);
2204         }
2205 }
2206
2207 static void
2208 bbr_log_rtt_sample(struct tcp_bbr *bbr, uint32_t rtt, uint32_t tsin)
2209 {
2210         /*
2211          * Log the rtt sample we are applying to the srtt algorithm in
2212          * useconds.
2213          */
2214         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2215                 union tcp_log_stackspecific log;
2216
2217                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2218                 log.u_bbr.flex1 = rtt;
2219                 log.u_bbr.flex2 = bbr->r_ctl.rc_bbr_state_time;
2220                 log.u_bbr.flex3 = bbr->r_ctl.rc_ack_hdwr_delay;
2221                 log.u_bbr.flex4 = bbr->rc_tp->ts_offset;
2222                 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2223                 log.u_bbr.pkts_out = tcp_tv_to_mssectick(&bbr->rc_tv);
2224                 log.u_bbr.flex6 = tsin;
2225                 log.u_bbr.flex7 = 0;
2226                 log.u_bbr.flex8 = bbr->rc_ack_was_delayed;
2227                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2228                     &bbr->rc_inp->inp_socket->so_rcv,
2229                     &bbr->rc_inp->inp_socket->so_snd,
2230                     TCP_LOG_RTT, 0,
2231                     0, &log, false, &bbr->rc_tv);
2232         }
2233 }
2234
2235 static void
2236 bbr_log_type_pesist(struct tcp_bbr *bbr, uint32_t cts, uint32_t time_in, int32_t line, uint8_t enter_exit)
2237 {
2238         if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2239                 union tcp_log_stackspecific log;
2240
2241                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2242                 log.u_bbr.flex1 = time_in;
2243                 log.u_bbr.flex2 = line;
2244                 log.u_bbr.flex8 = enter_exit;
2245                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2246                     &bbr->rc_inp->inp_socket->so_rcv,
2247                     &bbr->rc_inp->inp_socket->so_snd,
2248                     BBR_LOG_PERSIST, 0,
2249                     0, &log, false, &bbr->rc_tv);
2250         }
2251 }
2252 static void
2253 bbr_log_ack_clear(struct tcp_bbr *bbr, uint32_t cts)
2254 {
2255         if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2256                 union tcp_log_stackspecific log;
2257
2258                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2259                 log.u_bbr.flex1 = bbr->rc_tp->ts_recent_age;
2260                 log.u_bbr.flex2 = bbr->r_ctl.rc_rtt_shrinks;
2261                 log.u_bbr.flex3 = bbr->r_ctl.rc_probertt_int;
2262                 log.u_bbr.flex4 = bbr->r_ctl.rc_went_idle_time;
2263                 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2264                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2265                     &bbr->rc_inp->inp_socket->so_rcv,
2266                     &bbr->rc_inp->inp_socket->so_snd,
2267                     BBR_LOG_ACKCLEAR, 0,
2268                     0, &log, false, &bbr->rc_tv);
2269         }
2270 }
2271
2272 static void
2273 bbr_log_ack_event(struct tcp_bbr *bbr, struct tcphdr *th, struct tcpopt *to, uint32_t tlen,
2274                   uint16_t nsegs, uint32_t cts, int32_t nxt_pkt, struct mbuf *m)
2275 {
2276         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2277                 union tcp_log_stackspecific log;
2278                 struct timeval tv;
2279
2280                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2281                 log.u_bbr.flex1 = nsegs;
2282                 log.u_bbr.flex2 = bbr->r_ctl.rc_lost_bytes;
2283                 if (m) {
2284                         struct timespec ts;
2285
2286                         log.u_bbr.flex3 = m->m_flags;
2287                         if (m->m_flags & M_TSTMP) {
2288                                 mbuf_tstmp2timespec(m, &ts);
2289                                 tv.tv_sec = ts.tv_sec;
2290                                 tv.tv_usec = ts.tv_nsec / 1000;
2291                                 log.u_bbr.lt_epoch = tcp_tv_to_usectick(&tv);
2292                         } else {
2293                                 log.u_bbr.lt_epoch = 0;
2294                         }
2295                         if (m->m_flags & M_TSTMP_LRO) {
2296                                 tv.tv_sec = m->m_pkthdr.rcv_tstmp / 1000000000;
2297                                 tv.tv_usec = (m->m_pkthdr.rcv_tstmp % 1000000000) / 1000;
2298                                 log.u_bbr.flex5 = tcp_tv_to_usectick(&tv);
2299                         } else {
2300                                 /* No arrival timestamp */
2301                                 log.u_bbr.flex5 = 0;
2302                         }
2303
2304                         log.u_bbr.pkts_out = tcp_get_usecs(&tv);
2305                 } else {
2306                         log.u_bbr.flex3 = 0;
2307                         log.u_bbr.flex5 = 0;
2308                         log.u_bbr.flex6 = 0;
2309                         log.u_bbr.pkts_out = 0;
2310                 }
2311                 log.u_bbr.flex4 = bbr->r_ctl.rc_target_at_state;
2312                 log.u_bbr.flex7 = bbr->r_wanted_output;
2313                 log.u_bbr.flex8 = bbr->rc_in_persist;
2314                 TCP_LOG_EVENTP(bbr->rc_tp, th,
2315                     &bbr->rc_inp->inp_socket->so_rcv,
2316                     &bbr->rc_inp->inp_socket->so_snd,
2317                     TCP_LOG_IN, 0,
2318                     tlen, &log, true, &bbr->rc_tv);
2319         }
2320 }
2321
2322 static void
2323 bbr_log_doseg_done(struct tcp_bbr *bbr, uint32_t cts, int32_t nxt_pkt, int32_t did_out)
2324 {
2325         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2326                 union tcp_log_stackspecific log;
2327
2328                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2329                 log.u_bbr.flex1 = did_out;
2330                 log.u_bbr.flex2 = nxt_pkt;
2331                 log.u_bbr.flex3 = bbr->r_ctl.rc_last_delay_val;
2332                 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2333                 log.u_bbr.flex5 = bbr->r_ctl.rc_timer_exp;
2334                 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_bytes;
2335                 log.u_bbr.flex7 = bbr->r_wanted_output;
2336                 log.u_bbr.flex8 = bbr->rc_in_persist;
2337                 log.u_bbr.pkts_out = bbr->r_ctl.highest_hdwr_delay;
2338                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2339                     &bbr->rc_inp->inp_socket->so_rcv,
2340                     &bbr->rc_inp->inp_socket->so_snd,
2341                     BBR_LOG_DOSEG_DONE, 0,
2342                     0, &log, true, &bbr->rc_tv);
2343         }
2344 }
2345
2346 static void
2347 bbr_log_enobuf_jmp(struct tcp_bbr *bbr, uint32_t len, uint32_t cts,
2348     int32_t line, uint32_t o_len, uint32_t segcnt, uint32_t segsiz)
2349 {
2350         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2351                 union tcp_log_stackspecific log;
2352
2353                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2354                 log.u_bbr.flex1 = line;
2355                 log.u_bbr.flex2 = o_len;
2356                 log.u_bbr.flex3 = segcnt;
2357                 log.u_bbr.flex4 = segsiz;
2358                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2359                     &bbr->rc_inp->inp_socket->so_rcv,
2360                     &bbr->rc_inp->inp_socket->so_snd,
2361                     BBR_LOG_ENOBUF_JMP, ENOBUFS,
2362                     len, &log, true, &bbr->rc_tv);
2363         }
2364 }
2365
2366 static void
2367 bbr_log_to_processing(struct tcp_bbr *bbr, uint32_t cts, int32_t ret, int32_t timers, uint8_t hpts_calling)
2368 {
2369         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2370                 union tcp_log_stackspecific log;
2371
2372                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2373                 log.u_bbr.flex1 = timers;
2374                 log.u_bbr.flex2 = ret;
2375                 log.u_bbr.flex3 = bbr->r_ctl.rc_timer_exp;
2376                 log.u_bbr.flex4 = bbr->r_ctl.rc_hpts_flags;
2377                 log.u_bbr.flex5 = cts;
2378                 log.u_bbr.flex6 = bbr->r_ctl.rc_target_at_state;
2379                 log.u_bbr.flex8 = hpts_calling;
2380                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2381                     &bbr->rc_inp->inp_socket->so_rcv,
2382                     &bbr->rc_inp->inp_socket->so_snd,
2383                     BBR_LOG_TO_PROCESS, 0,
2384                     0, &log, false, &bbr->rc_tv);
2385         }
2386 }
2387
2388 static void
2389 bbr_log_to_event(struct tcp_bbr *bbr, uint32_t cts, int32_t to_num)
2390 {
2391         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2392                 union tcp_log_stackspecific log;
2393                 uint64_t ar;
2394
2395                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2396                 log.u_bbr.flex1 = bbr->bbr_timer_src;
2397                 log.u_bbr.flex2 = 0;
2398                 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2399                 ar = (uint64_t)(bbr->r_ctl.rc_resend);
2400                 ar >>= 32;
2401                 ar &= 0x00000000ffffffff;
2402                 log.u_bbr.flex4 = (uint32_t)ar;
2403                 ar = (uint64_t)bbr->r_ctl.rc_resend;
2404                 ar &= 0x00000000ffffffff;
2405                 log.u_bbr.flex5 = (uint32_t)ar;
2406                 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2407                 log.u_bbr.flex8 = to_num;
2408                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2409                     &bbr->rc_inp->inp_socket->so_rcv,
2410                     &bbr->rc_inp->inp_socket->so_snd,
2411                     BBR_LOG_RTO, 0,
2412                     0, &log, false, &bbr->rc_tv);
2413         }
2414 }
2415
2416 static void
2417 bbr_log_startup_event(struct tcp_bbr *bbr, uint32_t cts, uint32_t flex1, uint32_t flex2, uint32_t flex3, uint8_t reason)
2418 {
2419         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2420                 union tcp_log_stackspecific log;
2421
2422                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2423                 log.u_bbr.flex1 = flex1;
2424                 log.u_bbr.flex2 = flex2;
2425                 log.u_bbr.flex3 = flex3;
2426                 log.u_bbr.flex4 = 0;
2427                 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2428                 log.u_bbr.flex6 = bbr->r_ctl.rc_lost_at_startup;
2429                 log.u_bbr.flex8 = reason;
2430                 log.u_bbr.cur_del_rate = bbr->r_ctl.rc_bbr_lastbtlbw;
2431                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2432                     &bbr->rc_inp->inp_socket->so_rcv,
2433                     &bbr->rc_inp->inp_socket->so_snd,
2434                     BBR_LOG_REDUCE, 0,
2435                     0, &log, false, &bbr->rc_tv);
2436         }
2437 }
2438
2439 static void
2440 bbr_log_hpts_diag(struct tcp_bbr *bbr, uint32_t cts, struct hpts_diag *diag)
2441 {
2442         if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2443                 union tcp_log_stackspecific log;
2444
2445                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2446                 log.u_bbr.flex1 = diag->p_nxt_slot;
2447                 log.u_bbr.flex2 = diag->p_cur_slot;
2448                 log.u_bbr.flex3 = diag->slot_req;
2449                 log.u_bbr.flex4 = diag->inp_hptsslot;
2450                 log.u_bbr.flex5 = diag->slot_remaining;
2451                 log.u_bbr.flex6 = diag->need_new_to;
2452                 log.u_bbr.flex7 = diag->p_hpts_active;
2453                 log.u_bbr.flex8 = diag->p_on_min_sleep;
2454                 /* Hijack other fields as needed  */
2455                 log.u_bbr.epoch = diag->have_slept;
2456                 log.u_bbr.lt_epoch = diag->yet_to_sleep;
2457                 log.u_bbr.pkts_out = diag->co_ret;
2458                 log.u_bbr.applimited = diag->hpts_sleep_time;
2459                 log.u_bbr.delivered = diag->p_prev_slot;
2460                 log.u_bbr.inflight = diag->p_runningtick;
2461                 log.u_bbr.bw_inuse = diag->wheel_tick;
2462                 log.u_bbr.rttProp = diag->wheel_cts;
2463                 log.u_bbr.delRate = diag->maxticks;
2464                 log.u_bbr.cur_del_rate = diag->p_curtick;
2465                 log.u_bbr.cur_del_rate <<= 32;
2466                 log.u_bbr.cur_del_rate |= diag->p_lasttick;
2467                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2468                     &bbr->rc_inp->inp_socket->so_rcv,
2469                     &bbr->rc_inp->inp_socket->so_snd,
2470                     BBR_LOG_HPTSDIAG, 0,
2471                     0, &log, false, &bbr->rc_tv);
2472         }
2473 }
2474
2475 static void
2476 bbr_log_timer_var(struct tcp_bbr *bbr, int mode, uint32_t cts, uint32_t time_since_sent, uint32_t srtt,
2477     uint32_t thresh, uint32_t to)
2478 {
2479         if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2480                 union tcp_log_stackspecific log;
2481
2482                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2483                 log.u_bbr.flex1 = bbr->rc_tp->t_rttvar;
2484                 log.u_bbr.flex2 = time_since_sent;
2485                 log.u_bbr.flex3 = srtt;
2486                 log.u_bbr.flex4 = thresh;
2487                 log.u_bbr.flex5 = to;
2488                 log.u_bbr.flex6 = bbr->rc_tp->t_srtt;
2489                 log.u_bbr.flex8 = mode;
2490                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2491                     &bbr->rc_inp->inp_socket->so_rcv,
2492                     &bbr->rc_inp->inp_socket->so_snd,
2493                     BBR_LOG_TIMERPREP, 0,
2494                     0, &log, false, &bbr->rc_tv);
2495         }
2496 }
2497
2498 static void
2499 bbr_log_pacing_delay_calc(struct tcp_bbr *bbr, uint16_t gain, uint32_t len,
2500     uint32_t cts, uint32_t usecs, uint64_t bw, uint32_t override, int mod)
2501 {
2502         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2503                 union tcp_log_stackspecific log;
2504
2505                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2506                 log.u_bbr.flex1 = usecs;
2507                 log.u_bbr.flex2 = len;
2508                 log.u_bbr.flex3 = (uint32_t)((bw >> 32) & 0x00000000ffffffff);
2509                 log.u_bbr.flex4 = (uint32_t)(bw & 0x00000000ffffffff);
2510                 if (override)
2511                         log.u_bbr.flex5 = (1 << 2);
2512                 else
2513                         log.u_bbr.flex5 = 0;
2514                 log.u_bbr.flex6 = override;
2515                 log.u_bbr.flex7 = gain;
2516                 log.u_bbr.flex8 = mod;
2517                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2518                     &bbr->rc_inp->inp_socket->so_rcv,
2519                     &bbr->rc_inp->inp_socket->so_snd,
2520                     BBR_LOG_HPTSI_CALC, 0,
2521                     len, &log, false, &bbr->rc_tv);
2522         }
2523 }
2524
2525 static void
2526 bbr_log_to_start(struct tcp_bbr *bbr, uint32_t cts, uint32_t to, int32_t slot, uint8_t which)
2527 {
2528         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2529                 union tcp_log_stackspecific log;
2530
2531                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2532
2533                 log.u_bbr.flex1 = bbr->bbr_timer_src;
2534                 log.u_bbr.flex2 = to;
2535                 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2536                 log.u_bbr.flex4 = slot;
2537                 log.u_bbr.flex5 = bbr->rc_inp->inp_hptsslot;
2538                 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2539                 log.u_bbr.pkts_out = bbr->rc_inp->inp_flags2;
2540                 log.u_bbr.flex8 = which;
2541                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2542                     &bbr->rc_inp->inp_socket->so_rcv,
2543                     &bbr->rc_inp->inp_socket->so_snd,
2544                     BBR_LOG_TIMERSTAR, 0,
2545                     0, &log, false, &bbr->rc_tv);
2546         }
2547 }
2548
2549 static void
2550 bbr_log_thresh_choice(struct tcp_bbr *bbr, uint32_t cts, uint32_t thresh, uint32_t lro, uint32_t srtt, struct bbr_sendmap *rsm, uint8_t frm)
2551 {
2552         if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2553                 union tcp_log_stackspecific log;
2554
2555                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2556                 log.u_bbr.flex1 = thresh;
2557                 log.u_bbr.flex2 = lro;
2558                 log.u_bbr.flex3 = bbr->r_ctl.rc_reorder_ts;
2559                 log.u_bbr.flex4 = rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)];
2560                 log.u_bbr.flex5 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2561                 log.u_bbr.flex6 = srtt;
2562                 log.u_bbr.flex7 = bbr->r_ctl.rc_reorder_shift;
2563                 log.u_bbr.flex8 = frm;
2564                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2565                     &bbr->rc_inp->inp_socket->so_rcv,
2566                     &bbr->rc_inp->inp_socket->so_snd,
2567                     BBR_LOG_THRESH_CALC, 0,
2568                     0, &log, false, &bbr->rc_tv);
2569         }
2570 }
2571
2572 static void
2573 bbr_log_to_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts, uint8_t hpts_removed)
2574 {
2575         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2576                 union tcp_log_stackspecific log;
2577
2578                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2579                 log.u_bbr.flex1 = line;
2580                 log.u_bbr.flex2 = bbr->bbr_timer_src;
2581                 log.u_bbr.flex3 = bbr->r_ctl.rc_hpts_flags;
2582                 log.u_bbr.flex4 = bbr->rc_in_persist;
2583                 log.u_bbr.flex5 = bbr->r_ctl.rc_target_at_state;
2584                 log.u_bbr.flex6 = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
2585                 log.u_bbr.flex8 = hpts_removed;
2586                 log.u_bbr.pkts_out = bbr->rc_pacer_started;
2587                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2588                     &bbr->rc_inp->inp_socket->so_rcv,
2589                     &bbr->rc_inp->inp_socket->so_snd,
2590                     BBR_LOG_TIMERCANC, 0,
2591                     0, &log, false, &bbr->rc_tv);
2592         }
2593 }
2594
2595
2596 static void
2597 bbr_log_tstmp_validation(struct tcp_bbr *bbr, uint64_t peer_delta, uint64_t delta)
2598 {
2599         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2600                 union tcp_log_stackspecific log;
2601
2602                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2603                 log.u_bbr.flex1 = bbr->r_ctl.bbr_peer_tsratio;
2604                 log.u_bbr.flex2 = (peer_delta >> 32);
2605                 log.u_bbr.flex3 = (peer_delta & 0x00000000ffffffff);
2606                 log.u_bbr.flex4 = (delta >> 32);
2607                 log.u_bbr.flex5 = (delta & 0x00000000ffffffff);
2608                 log.u_bbr.flex7 = bbr->rc_ts_clock_set;
2609                 log.u_bbr.flex8 = bbr->rc_ts_cant_be_used;
2610                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2611                     &bbr->rc_inp->inp_socket->so_rcv,
2612                     &bbr->rc_inp->inp_socket->so_snd,
2613                     BBR_LOG_TSTMP_VAL, 0,
2614                     0, &log, false, &bbr->rc_tv);
2615
2616         }
2617 }
2618
2619 static void
2620 bbr_log_type_tsosize(struct tcp_bbr *bbr, uint32_t cts, uint32_t tsosz, uint32_t tls, uint32_t old_val, uint32_t maxseg, int hdwr)
2621 {
2622         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2623                 union tcp_log_stackspecific log;
2624
2625                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2626                 log.u_bbr.flex1 = tsosz;
2627                 log.u_bbr.flex2 = tls;
2628                 log.u_bbr.flex3 = tcp_min_hptsi_time;
2629                 log.u_bbr.flex4 = bbr->r_ctl.bbr_hptsi_bytes_min;
2630                 log.u_bbr.flex5 = old_val;
2631                 log.u_bbr.flex6 = maxseg;
2632                 log.u_bbr.flex7 = bbr->rc_no_pacing;
2633                 log.u_bbr.flex7 <<= 1;
2634                 log.u_bbr.flex7 |= bbr->rc_past_init_win;
2635                 if (hdwr)
2636                         log.u_bbr.flex8 = 0x80 | bbr->rc_use_google;
2637                 else
2638                         log.u_bbr.flex8 = bbr->rc_use_google;
2639                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2640                     &bbr->rc_inp->inp_socket->so_rcv,
2641                     &bbr->rc_inp->inp_socket->so_snd,
2642                     BBR_LOG_BBRTSO, 0,
2643                     0, &log, false, &bbr->rc_tv);
2644         }
2645 }
2646
2647 static void
2648 bbr_log_type_rsmclear(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm,
2649                       uint32_t flags, uint32_t line)
2650 {
2651         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2652                 union tcp_log_stackspecific log;
2653
2654                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2655                 log.u_bbr.flex1 = line;
2656                 log.u_bbr.flex2 = rsm->r_start;
2657                 log.u_bbr.flex3 = rsm->r_end;
2658                 log.u_bbr.flex4 = rsm->r_delivered;
2659                 log.u_bbr.flex5 = rsm->r_rtr_cnt;
2660                 log.u_bbr.flex6 = rsm->r_dupack;
2661                 log.u_bbr.flex7 = rsm->r_tim_lastsent[0];
2662                 log.u_bbr.flex8 = rsm->r_flags;
2663                 /* Hijack the pkts_out fids */
2664                 log.u_bbr.applimited = flags;
2665                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2666                     &bbr->rc_inp->inp_socket->so_rcv,
2667                     &bbr->rc_inp->inp_socket->so_snd,
2668                     BBR_RSM_CLEARED, 0,
2669                     0, &log, false, &bbr->rc_tv);
2670         }
2671 }
2672
2673 static void
2674 bbr_log_type_bbrupd(struct tcp_bbr *bbr, uint8_t flex8, uint32_t cts,
2675     uint32_t flex3, uint32_t flex2, uint32_t flex5,
2676     uint32_t flex6, uint32_t pkts_out, int flex7,
2677     uint32_t flex4, uint32_t flex1)
2678 {
2679
2680         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2681                 union tcp_log_stackspecific log;
2682
2683                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2684                 log.u_bbr.flex1 = flex1;
2685                 log.u_bbr.flex2 = flex2;
2686                 log.u_bbr.flex3 = flex3;
2687                 log.u_bbr.flex4 = flex4;
2688                 log.u_bbr.flex5 = flex5;
2689                 log.u_bbr.flex6 = flex6;
2690                 log.u_bbr.flex7 = flex7;
2691                 /* Hijack the pkts_out fids */
2692                 log.u_bbr.pkts_out = pkts_out;
2693                 log.u_bbr.flex8 = flex8;
2694                 if (bbr->rc_ack_was_delayed)
2695                         log.u_bbr.epoch = bbr->r_ctl.rc_ack_hdwr_delay;
2696                 else
2697                         log.u_bbr.epoch = 0;
2698                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2699                     &bbr->rc_inp->inp_socket->so_rcv,
2700                     &bbr->rc_inp->inp_socket->so_snd,
2701                     BBR_LOG_BBRUPD, 0,
2702                     flex2, &log, false, &bbr->rc_tv);
2703         }
2704 }
2705
2706
2707 static void
2708 bbr_log_type_ltbw(struct tcp_bbr *bbr, uint32_t cts, int32_t reason,
2709         uint32_t newbw, uint32_t obw, uint32_t diff,
2710         uint32_t tim)
2711 {
2712         if (/*bbr_verbose_logging && */(bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2713                 union tcp_log_stackspecific log;
2714
2715                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2716                 log.u_bbr.flex1 = reason;
2717                 log.u_bbr.flex2 = newbw;
2718                 log.u_bbr.flex3 = obw;
2719                 log.u_bbr.flex4 = diff;
2720                 log.u_bbr.flex5 = bbr->r_ctl.rc_lt_lost;
2721                 log.u_bbr.flex6 = bbr->r_ctl.rc_lt_del;
2722                 log.u_bbr.flex7 = bbr->rc_lt_is_sampling;
2723                 log.u_bbr.pkts_out = tim;
2724                 log.u_bbr.bw_inuse = bbr->r_ctl.rc_lt_bw;
2725                 if (bbr->rc_lt_use_bw == 0)
2726                         log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
2727                 else
2728                         log.u_bbr.epoch = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
2729                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2730                     &bbr->rc_inp->inp_socket->so_rcv,
2731                     &bbr->rc_inp->inp_socket->so_snd,
2732                     BBR_LOG_BWSAMP, 0,
2733                     0, &log, false, &bbr->rc_tv);
2734         }
2735 }
2736
2737 static inline void
2738 bbr_log_progress_event(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t tick, int event, int line)
2739 {
2740         if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2741                 union tcp_log_stackspecific log;
2742
2743                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2744                 log.u_bbr.flex1 = line;
2745                 log.u_bbr.flex2 = tick;
2746                 log.u_bbr.flex3 = tp->t_maxunacktime;
2747                 log.u_bbr.flex4 = tp->t_acktime;
2748                 log.u_bbr.flex8 = event;
2749                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2750                     &bbr->rc_inp->inp_socket->so_rcv,
2751                     &bbr->rc_inp->inp_socket->so_snd,
2752                     BBR_LOG_PROGRESS, 0,
2753                     0, &log, false, &bbr->rc_tv);
2754         }
2755 }
2756
2757 static void
2758 bbr_type_log_hdwr_pacing(struct tcp_bbr *bbr, const struct ifnet *ifp,
2759                          uint64_t rate, uint64_t hw_rate, int line, uint32_t cts,
2760                          int error)
2761 {
2762         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2763                 union tcp_log_stackspecific log;
2764
2765                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2766                 log.u_bbr.flex1 = ((hw_rate >> 32) & 0x00000000ffffffff);
2767                 log.u_bbr.flex2 = (hw_rate & 0x00000000ffffffff);
2768                 log.u_bbr.flex3 = (((uint64_t)ifp  >> 32) & 0x00000000ffffffff);
2769                 log.u_bbr.flex4 = ((uint64_t)ifp & 0x00000000ffffffff);
2770                 log.u_bbr.bw_inuse = rate;
2771                 log.u_bbr.flex5 = line;
2772                 log.u_bbr.flex6 = error;
2773                 log.u_bbr.flex8 = bbr->skip_gain;
2774                 log.u_bbr.flex8 <<= 1;
2775                 log.u_bbr.flex8 |= bbr->gain_is_limited;
2776                 log.u_bbr.flex8 <<= 1;
2777                 log.u_bbr.flex8 |= bbr->bbr_hdrw_pacing;
2778                 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
2779                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2780                     &bbr->rc_inp->inp_socket->so_rcv,
2781                     &bbr->rc_inp->inp_socket->so_snd,
2782                     BBR_LOG_HDWR_PACE, 0,
2783                     0, &log, false, &bbr->rc_tv);
2784         }
2785 }
2786
2787 static void
2788 bbr_log_type_bbrsnd(struct tcp_bbr *bbr, uint32_t len, uint32_t slot, uint32_t del_by, uint32_t cts, uint32_t line, uint32_t prev_delay)
2789 {
2790         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2791                 union tcp_log_stackspecific log;
2792
2793                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2794                 log.u_bbr.flex1 = slot;
2795                 log.u_bbr.flex2 = del_by;
2796                 log.u_bbr.flex3 = prev_delay;
2797                 log.u_bbr.flex4 = line;
2798                 log.u_bbr.flex5 = bbr->r_ctl.rc_last_delay_val;
2799                 log.u_bbr.flex6 = bbr->r_ctl.rc_hptsi_agg_delay;
2800                 log.u_bbr.flex7 = (0x0000ffff & bbr->r_ctl.rc_hpts_flags);
2801                 log.u_bbr.flex8 = bbr->rc_in_persist;
2802                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2803                     &bbr->rc_inp->inp_socket->so_rcv,
2804                     &bbr->rc_inp->inp_socket->so_snd,
2805                     BBR_LOG_BBRSND, 0,
2806                     len, &log, false, &bbr->rc_tv);
2807         }
2808 }
2809
2810 static void
2811 bbr_log_type_bbrrttprop(struct tcp_bbr *bbr, uint32_t t, uint32_t end, uint32_t tsconv, uint32_t cts, int32_t match, uint32_t seq, uint8_t flags)
2812 {
2813         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2814                 union tcp_log_stackspecific log;
2815
2816                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2817                 log.u_bbr.flex1 = bbr->r_ctl.rc_delivered;
2818                 log.u_bbr.flex2 = 0;
2819                 log.u_bbr.flex3 = bbr->r_ctl.rc_lowest_rtt;
2820                 log.u_bbr.flex4 = end;
2821                 log.u_bbr.flex5 = seq;
2822                 log.u_bbr.flex6 = t;
2823                 log.u_bbr.flex7 = match;
2824                 log.u_bbr.flex8 = flags;
2825                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2826                     &bbr->rc_inp->inp_socket->so_rcv,
2827                     &bbr->rc_inp->inp_socket->so_snd,
2828                     BBR_LOG_BBRRTT, 0,
2829                     0, &log, false, &bbr->rc_tv);
2830         }
2831 }
2832
2833 static void
2834 bbr_log_exit_gain(struct tcp_bbr *bbr, uint32_t cts, int32_t entry_method)
2835 {
2836         if (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF) {
2837                 union tcp_log_stackspecific log;
2838
2839                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
2840                 log.u_bbr.flex1 = bbr->r_ctl.rc_target_at_state;
2841                 log.u_bbr.flex2 = (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
2842                 log.u_bbr.flex3 = bbr->r_ctl.gain_epoch;
2843                 log.u_bbr.flex4 = bbr->r_ctl.rc_pace_max_segs;
2844                 log.u_bbr.flex5 = bbr->r_ctl.rc_pace_min_segs;
2845                 log.u_bbr.flex6 = bbr->r_ctl.rc_bbr_state_atflight;
2846                 log.u_bbr.flex7 = 0;
2847                 log.u_bbr.flex8 = entry_method;
2848                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2849                     &bbr->rc_inp->inp_socket->so_rcv,
2850                     &bbr->rc_inp->inp_socket->so_snd,
2851                     BBR_LOG_EXIT_GAIN, 0,
2852                     0, &log, false, &bbr->rc_tv);
2853         }
2854 }
2855
2856 static void
2857 bbr_log_settings_change(struct tcp_bbr *bbr, int settings_desired)
2858 {
2859         if (bbr_verbose_logging && (bbr->rc_tp->t_logstate != TCP_LOG_STATE_OFF)) {
2860                 union tcp_log_stackspecific log;
2861
2862                 bbr_fill_in_logging_data(bbr, &log.u_bbr, bbr->r_ctl.rc_rcvtime);
2863                 /* R-HU */
2864                 log.u_bbr.flex1 = 0;
2865                 log.u_bbr.flex2 = 0;
2866                 log.u_bbr.flex3 = 0;
2867                 log.u_bbr.flex4 = 0;
2868                 log.u_bbr.flex7 = 0;
2869                 log.u_bbr.flex8 = settings_desired;
2870
2871                 TCP_LOG_EVENTP(bbr->rc_tp, NULL,
2872                     &bbr->rc_inp->inp_socket->so_rcv,
2873                     &bbr->rc_inp->inp_socket->so_snd,
2874                     BBR_LOG_SETTINGS_CHG, 0,
2875                     0, &log, false, &bbr->rc_tv);
2876         }
2877 }
2878
2879 /*
2880  * Returns the bw from the our filter.
2881  */
2882 static inline uint64_t
2883 bbr_get_full_bw(struct tcp_bbr *bbr)
2884 {
2885         uint64_t bw;
2886
2887         bw = get_filter_value(&bbr->r_ctl.rc_delrate);
2888
2889         return (bw);
2890 }
2891
2892 static inline void
2893 bbr_set_pktepoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2894 {
2895         uint64_t calclr;
2896         uint32_t lost, del;
2897
2898         if (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_pktepoch)
2899                 lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lost_at_pktepoch;
2900         else
2901                 lost = 0;
2902         del = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_pkt_epoch_del;
2903         if (lost == 0)  {
2904                 calclr = 0;
2905         } else if (del) {
2906                 calclr = lost;
2907                 calclr *= (uint64_t)1000;
2908                 calclr /= (uint64_t)del;
2909         } else {
2910                 /* Nothing delivered? 100.0% loss */
2911                 calclr = 1000;
2912         }
2913         bbr->r_ctl.rc_pkt_epoch_loss_rate =  (uint32_t)calclr;
2914         if (IN_RECOVERY(bbr->rc_tp->t_flags))
2915                 bbr->r_ctl.recovery_lr += (uint32_t)calclr;
2916         bbr->r_ctl.rc_pkt_epoch++;
2917         if (bbr->rc_no_pacing &&
2918             (bbr->r_ctl.rc_pkt_epoch >= bbr->no_pacing_until)) {
2919                 bbr->rc_no_pacing = 0;
2920                 tcp_bbr_tso_size_check(bbr, cts);
2921         }
2922         bbr->r_ctl.rc_pkt_epoch_rtt = bbr_calc_time(cts, bbr->r_ctl.rc_pkt_epoch_time);
2923         bbr->r_ctl.rc_pkt_epoch_time = cts;
2924         /* What was our loss rate */
2925         bbr_log_pkt_epoch(bbr, cts, line, lost, del);
2926         bbr->r_ctl.rc_pkt_epoch_del = bbr->r_ctl.rc_delivered;
2927         bbr->r_ctl.rc_lost_at_pktepoch = bbr->r_ctl.rc_lost;
2928 }
2929
2930 static inline void
2931 bbr_set_epoch(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
2932 {
2933         uint32_t epoch_time;
2934
2935         /* Tick the RTT clock */
2936         bbr->r_ctl.rc_rtt_epoch++;
2937         epoch_time = cts - bbr->r_ctl.rc_rcv_epoch_start;
2938         bbr_log_time_epoch(bbr, cts, line, epoch_time);
2939         bbr->r_ctl.rc_rcv_epoch_start = cts;
2940 }
2941
2942
2943 static inline void
2944 bbr_isit_a_pkt_epoch(struct tcp_bbr *bbr, uint32_t cts, struct bbr_sendmap *rsm, int32_t line, int32_t cum_acked)
2945 {
2946         if (SEQ_GEQ(rsm->r_delivered, bbr->r_ctl.rc_pkt_epoch_del)) {
2947                 bbr->rc_is_pkt_epoch_now = 1;
2948         }
2949 }
2950
2951 /*
2952  * Returns the bw from either the b/w filter
2953  * or from the lt_bw (if the connection is being
2954  * policed).
2955  */
2956 static inline uint64_t
2957 __bbr_get_bw(struct tcp_bbr *bbr)
2958 {
2959         uint64_t bw, min_bw;
2960         uint64_t rtt;
2961         int gm_measure_cnt = 1;
2962
2963         /*
2964          * For startup we make, like google, a
2965          * minimum b/w. This is generated from the
2966          * IW and the rttProp. We do fall back to srtt
2967          * if for some reason (initial handshake) we don't
2968          * have a rttProp. We, in the worst case, fall back
2969          * to the configured min_bw (rc_initial_hptsi_bw).
2970          */
2971         if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
2972                 /* Attempt first to use rttProp */
2973                 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
2974                 if (rtt && (rtt < 0xffffffff)) {
2975 measure:
2976                         min_bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
2977                                 ((uint64_t)1000000);
2978                         min_bw /= rtt;
2979                         if (min_bw < bbr->r_ctl.rc_initial_hptsi_bw) {
2980                                 min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2981                         }
2982
2983                 } else if (bbr->rc_tp->t_srtt != 0) {
2984                         /* No rttProp, use srtt? */
2985                         rtt = bbr_get_rtt(bbr, BBR_SRTT);
2986                         goto measure;
2987                 } else {
2988                         min_bw = bbr->r_ctl.rc_initial_hptsi_bw;
2989                 }
2990         } else
2991                 min_bw = 0;
2992
2993         if ((bbr->rc_past_init_win == 0) &&
2994             (bbr->r_ctl.rc_delivered > bbr_initial_cwnd(bbr, bbr->rc_tp)))
2995                 bbr->rc_past_init_win = 1;
2996         if ((bbr->rc_use_google)  && (bbr->r_ctl.r_measurement_count >= 1))
2997                 gm_measure_cnt = 0;
2998         if (gm_measure_cnt &&
2999             ((bbr->r_ctl.r_measurement_count < bbr_min_measurements_req) ||
3000              (bbr->rc_past_init_win == 0))) {
3001                 /* For google we use our guess rate until we get 1 measurement */
3002
3003 use_initial_window:
3004                 rtt = (uint64_t)get_filter_value_small(&bbr->r_ctl.rc_rttprop);
3005                 if (rtt && (rtt < 0xffffffff)) {
3006                         /*
3007                          * We have an RTT measurment. Use that in
3008                          * combination with our initial window to calculate
3009                          * a b/w.
3010                          */
3011                         bw = (uint64_t)(bbr_initial_cwnd(bbr, bbr->rc_tp)) *
3012                                 ((uint64_t)1000000);
3013                         bw /= rtt;
3014                         if (bw < bbr->r_ctl.rc_initial_hptsi_bw) {
3015                                 bw = bbr->r_ctl.rc_initial_hptsi_bw;
3016                         }
3017                 } else {
3018                         /* Drop back to the 40 and punt to a default */
3019                         bw = bbr->r_ctl.rc_initial_hptsi_bw;
3020                 }
3021                 if (bw < 1)
3022                         /* Probably should panic */
3023                         bw = 1;
3024                 if (bw > min_bw)
3025                         return (bw);
3026                 else
3027                         return (min_bw);
3028         }
3029         if (bbr->rc_lt_use_bw)
3030                 bw = bbr->r_ctl.rc_lt_bw;
3031         else if (bbr->r_recovery_bw && (bbr->rc_use_google == 0))
3032                 bw = bbr->r_ctl.red_bw;
3033         else
3034                 bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3035         if (bbr->rc_tp->t_peakrate_thr && (bbr->rc_use_google == 0)) {
3036                 /*
3037                  * Enforce user set rate limit, keep in mind that
3038                  * t_peakrate_thr is in B/s already
3039                  */
3040                 bw = uqmin((uint64_t)bbr->rc_tp->t_peakrate_thr, bw);
3041         }
3042         if (bw == 0) {
3043                 /* We should not be at 0, go to the initial window then  */
3044                 goto use_initial_window;
3045         }
3046         if (bw < 1)
3047                 /* Probably should panic */
3048                 bw = 1;
3049         if (bw < min_bw)
3050                 bw = min_bw;
3051         return (bw);
3052 }
3053
3054 static inline uint64_t
3055 bbr_get_bw(struct tcp_bbr *bbr)
3056 {
3057         uint64_t bw;
3058
3059         bw = __bbr_get_bw(bbr);
3060         return (bw);
3061 }
3062
3063 static inline void
3064 bbr_reset_lt_bw_interval(struct tcp_bbr *bbr, uint32_t cts)
3065 {
3066         bbr->r_ctl.rc_lt_epoch = bbr->r_ctl.rc_pkt_epoch;
3067         bbr->r_ctl.rc_lt_time = bbr->r_ctl.rc_del_time;
3068         bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3069         bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3070 }
3071
3072 static inline void
3073 bbr_reset_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts)
3074 {
3075         bbr->rc_lt_is_sampling = 0;
3076         bbr->rc_lt_use_bw = 0;
3077         bbr->r_ctl.rc_lt_bw = 0;
3078         bbr_reset_lt_bw_interval(bbr, cts);
3079 }
3080
3081 static inline void
3082 bbr_lt_bw_samp_done(struct tcp_bbr *bbr, uint64_t bw, uint32_t cts, uint32_t timin)
3083 {
3084         uint64_t diff;
3085
3086         /* Do we have a previous sample? */
3087         if (bbr->r_ctl.rc_lt_bw) {
3088                 /* Get the diff in bytes per second */
3089                 if (bbr->r_ctl.rc_lt_bw > bw)
3090                         diff = bbr->r_ctl.rc_lt_bw - bw;
3091                 else
3092                         diff = bw - bbr->r_ctl.rc_lt_bw;
3093                 if ((diff <= bbr_lt_bw_diff) ||
3094                     (diff <= (bbr->r_ctl.rc_lt_bw / bbr_lt_bw_ratio))) {
3095                         /* Consider us policed */
3096                         uint32_t saved_bw;
3097
3098                         saved_bw = (uint32_t)bbr->r_ctl.rc_lt_bw;
3099                         bbr->r_ctl.rc_lt_bw = (bw + bbr->r_ctl.rc_lt_bw) / 2;   /* average of two */
3100                         bbr->rc_lt_use_bw = 1;
3101                         bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
3102                         /*
3103                          * Use pkt based epoch for measuring length of
3104                          * policer up
3105                          */
3106                         bbr->r_ctl.rc_lt_epoch_use = bbr->r_ctl.rc_pkt_epoch;
3107                         /*
3108                          * reason 4 is we need to start consider being
3109                          * policed
3110                          */
3111                         bbr_log_type_ltbw(bbr, cts, 4, (uint32_t)bw, saved_bw, (uint32_t)diff, timin);
3112                         return;
3113                 }
3114         }
3115         bbr->r_ctl.rc_lt_bw = bw;
3116         bbr_reset_lt_bw_interval(bbr, cts);
3117         bbr_log_type_ltbw(bbr, cts, 5, 0, (uint32_t)bw, 0, timin);
3118 }
3119
3120 /*
3121  * RRS: Copied from user space!
3122  * Calculate a uniformly distributed random number less than upper_bound
3123  * avoiding "modulo bias".
3124  *
3125  * Uniformity is achieved by generating new random numbers until the one
3126  * returned is outside the range [0, 2**32 % upper_bound).  This
3127  * guarantees the selected random number will be inside
3128  * [2**32 % upper_bound, 2**32) which maps back to [0, upper_bound)
3129  * after reduction modulo upper_bound.
3130  */
3131 static uint32_t
3132 arc4random_uniform(uint32_t upper_bound)
3133 {
3134         uint32_t r, min;
3135
3136         if (upper_bound < 2)
3137                 return 0;
3138
3139         /* 2**32 % x == (2**32 - x) % x */
3140         min = -upper_bound % upper_bound;
3141
3142         /*
3143          * This could theoretically loop forever but each retry has
3144          * p > 0.5 (worst case, usually far better) of selecting a
3145          * number inside the range we need, so it should rarely need
3146          * to re-roll.
3147          */
3148         for (;;) {
3149                 r = arc4random();
3150                 if (r >= min)
3151                         break;
3152         }
3153
3154         return r % upper_bound;
3155 }
3156
3157 static void
3158 bbr_randomize_extra_state_time(struct tcp_bbr *bbr)
3159 {
3160         uint32_t ran, deduct;
3161
3162         ran = arc4random_uniform(bbr_rand_ot);
3163         if (ran) {
3164                 deduct = bbr->r_ctl.rc_level_state_extra / ran;
3165                 bbr->r_ctl.rc_level_state_extra -= deduct;
3166         }
3167 }
3168 /*
3169  * Return randomly the starting state
3170  * to use in probebw.
3171  */
3172 static uint8_t
3173 bbr_pick_probebw_substate(struct tcp_bbr *bbr, uint32_t cts)
3174 {
3175         uint32_t ran;
3176         uint8_t ret_val;
3177
3178         /* Initialize the offset to 0 */
3179         bbr->r_ctl.rc_exta_time_gd = 0;
3180         bbr->rc_hit_state_1 = 0;
3181         bbr->r_ctl.rc_level_state_extra = 0;
3182         ran = arc4random_uniform((BBR_SUBSTATE_COUNT-1));
3183         /*
3184          * The math works funny here :) the return value is used to set the
3185          * substate and then the state change is called which increments by
3186          * one. So if we return 1 (DRAIN) we will increment to 2 (LEVEL1) when
3187          * we fully enter the state. Note that the (8 - 1 - ran) assures that
3188          * we return 1 - 7, so we dont return 0 and end up starting in
3189          * state 1 (DRAIN).
3190          */
3191         ret_val = BBR_SUBSTATE_COUNT - 1 - ran;
3192         /* Set an epoch */
3193         if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP))
3194                 bbr_set_epoch(bbr, cts, __LINE__);
3195
3196         bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
3197         return (ret_val);
3198 }
3199
3200 static void
3201 bbr_lt_bw_sampling(struct tcp_bbr *bbr, uint32_t cts, int32_t loss_detected)
3202 {
3203         uint32_t diff, d_time;
3204         uint64_t del_time, bw, lost, delivered;
3205
3206         if (bbr->r_use_policer == 0)
3207                 return;
3208         if (bbr->rc_lt_use_bw) {
3209                 /* We are using lt bw do we stop yet? */
3210                 diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch_use;
3211                 if (diff > bbr_lt_bw_max_rtts) {
3212                         /* Reset it all */
3213 reset_all:
3214                         bbr_reset_lt_bw_sampling(bbr, cts);
3215                         if (bbr->rc_filled_pipe) {
3216                                 bbr_set_epoch(bbr, cts, __LINE__);
3217                                 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
3218                                 bbr_substate_change(bbr, cts, __LINE__, 0);
3219                                 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
3220                                 bbr_log_type_statechange(bbr, cts, __LINE__);
3221                         } else {
3222                                 /*
3223                                  * This should not happen really
3224                                  * unless we remove the startup/drain
3225                                  * restrictions above.
3226                                  */
3227                                 bbr->rc_bbr_state = BBR_STATE_STARTUP;
3228                                 bbr_set_epoch(bbr, cts, __LINE__);
3229                                 bbr->r_ctl.rc_bbr_state_time = cts;
3230                                 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
3231                                 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
3232                                 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
3233                                 bbr_set_state_target(bbr, __LINE__);
3234                                 bbr_log_type_statechange(bbr, cts, __LINE__);
3235                         }
3236                         /* reason 0 is to stop using lt-bw */
3237                         bbr_log_type_ltbw(bbr, cts, 0, 0, 0, 0, 0);
3238                         return;
3239                 }
3240                 if (bbr_lt_intvl_fp == 0) {
3241                         /* Not doing false-postive detection */
3242                         return;
3243                 }
3244                 /* False positive detection */
3245                 if (diff == bbr_lt_intvl_fp) {
3246                         /* At bbr_lt_intvl_fp we record the lost */
3247                         bbr->r_ctl.rc_lt_del = bbr->r_ctl.rc_delivered;
3248                         bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
3249                 } else if (diff > (bbr_lt_intvl_min_rtts + bbr_lt_intvl_fp)) {
3250                         /* Now is our loss rate still high? */
3251                         lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3252                         delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3253                         if ((delivered == 0) ||
3254                             (((lost * 1000)/delivered) < bbr_lt_fd_thresh)) {
3255                                 /* No still below our threshold */
3256                                 bbr_log_type_ltbw(bbr, cts, 7, lost, delivered, 0, 0);
3257                         } else {
3258                                 /* Yikes its still high, it must be a false positive */
3259                                 bbr_log_type_ltbw(bbr, cts, 8, lost, delivered, 0, 0);
3260                                 goto reset_all;
3261                         }
3262                 }
3263                 return;
3264         }
3265         /*
3266          * Wait for the first loss before sampling, to let the policer
3267          * exhaust its tokens and estimate the steady-state rate allowed by
3268          * the policer. Starting samples earlier includes bursts that
3269          * over-estimate the bw.
3270          */
3271         if (bbr->rc_lt_is_sampling == 0) {
3272                 /* reason 1 is to begin doing the sampling  */
3273                 if (loss_detected == 0)
3274                         return;
3275                 bbr_reset_lt_bw_interval(bbr, cts);
3276                 bbr->rc_lt_is_sampling = 1;
3277                 bbr_log_type_ltbw(bbr, cts, 1, 0, 0, 0, 0);
3278                 return;
3279         }
3280         /* Now how long were we delivering long term last> */
3281         if (TSTMP_GEQ(bbr->r_ctl.rc_del_time, bbr->r_ctl.rc_lt_time))
3282                 d_time = bbr->r_ctl.rc_del_time - bbr->r_ctl.rc_lt_time;
3283         else
3284                 d_time = 0;
3285
3286         /* To avoid underestimates, reset sampling if we run out of data. */
3287         if (bbr->r_ctl.r_app_limited_until) {
3288                 /* Can not measure in app-limited state */
3289                 bbr_reset_lt_bw_sampling(bbr, cts);
3290                 /* reason 2 is to reset sampling due to app limits  */
3291                 bbr_log_type_ltbw(bbr, cts, 2, 0, 0, 0, d_time);
3292                 return;
3293         }
3294         diff = bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_lt_epoch;
3295         if (diff < bbr_lt_intvl_min_rtts) {
3296                 /*
3297                  * need more samples (we don't
3298                  * start on a round like linux so
3299                  * we need 1 more).
3300                  */
3301                 /* 6 is not_enough time or no-loss */
3302                 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3303                 return;
3304         }
3305         if (diff > (4 * bbr_lt_intvl_min_rtts)) {
3306                 /*
3307                  * For now if we wait too long, reset all sampling. We need
3308                  * to do some research here, its possible that we should
3309                  * base this on how much loss as occurred.. something like
3310                  * if its under 10% (or some thresh) reset all otherwise
3311                  * don't.  Thats for phase II I guess.
3312                  */
3313                 bbr_reset_lt_bw_sampling(bbr, cts);
3314                 /* reason 3 is to reset sampling due too long of sampling */
3315                 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time);
3316                 return;
3317         }
3318         /*
3319          * End sampling interval when a packet is lost, so we estimate the
3320          * policer tokens were exhausted. Stopping the sampling before the
3321          * tokens are exhausted under-estimates the policed rate.
3322          */
3323         if (loss_detected == 0) {
3324                 /* 6 is not_enough time or no-loss */
3325                 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3326                 return;
3327         }
3328         /* Calculate packets lost and delivered in sampling interval. */
3329         lost = bbr->r_ctl.rc_lost - bbr->r_ctl.rc_lt_lost;
3330         delivered = bbr->r_ctl.rc_delivered - bbr->r_ctl.rc_lt_del;
3331         if ((delivered == 0) ||
3332             (((lost * 1000)/delivered) < bbr_lt_loss_thresh)) {
3333                 bbr_log_type_ltbw(bbr, cts, 6, lost, delivered, 0, d_time);
3334                 return;
3335         }
3336         if (d_time < 1000) {
3337                 /* Not enough time. wait */
3338                 /* 6 is not_enough time or no-loss */
3339                 bbr_log_type_ltbw(bbr, cts, 6, 0, 0, 0, d_time);
3340                 return;
3341         }
3342         if (d_time >= (0xffffffff / USECS_IN_MSEC)) {
3343                 /* Too long */
3344                 bbr_reset_lt_bw_sampling(bbr, cts);
3345                 /* reason 3 is to reset sampling due too long of sampling */
3346                 bbr_log_type_ltbw(bbr, cts, 3, 0, 0, 0, d_time);
3347                 return;
3348         }
3349         del_time = d_time;
3350         bw = delivered;
3351         bw *= (uint64_t)USECS_IN_SECOND;
3352         bw /= del_time;
3353         bbr_lt_bw_samp_done(bbr, bw, cts, d_time);
3354 }
3355
3356 /*
3357  * Allocate a sendmap from our zone.
3358  */
3359 static struct bbr_sendmap *
3360 bbr_alloc(struct tcp_bbr *bbr)
3361 {
3362         struct bbr_sendmap *rsm;
3363
3364         BBR_STAT_INC(bbr_to_alloc);
3365         rsm = uma_zalloc(bbr_zone, (M_NOWAIT | M_ZERO));
3366         if (rsm) {
3367                 bbr->r_ctl.rc_num_maps_alloced++;
3368                 return (rsm);
3369         }
3370         if (bbr->r_ctl.rc_free_cnt) {
3371                 BBR_STAT_INC(bbr_to_alloc_emerg);
3372                 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
3373                 TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
3374                 bbr->r_ctl.rc_free_cnt--;
3375                 return (rsm);
3376         }
3377         BBR_STAT_INC(bbr_to_alloc_failed);
3378         return (NULL);
3379 }
3380
3381 static struct bbr_sendmap *
3382 bbr_alloc_full_limit(struct tcp_bbr *bbr)
3383 {
3384         if ((V_tcp_map_entries_limit > 0) &&
3385             (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
3386                 BBR_STAT_INC(bbr_alloc_limited);
3387                 if (!bbr->alloc_limit_reported) {
3388                         bbr->alloc_limit_reported = 1;
3389                         BBR_STAT_INC(bbr_alloc_limited_conns);
3390                 }
3391                 return (NULL);
3392         }
3393         return (bbr_alloc(bbr));
3394 }
3395
3396
3397 /* wrapper to allocate a sendmap entry, subject to a specific limit */
3398 static struct bbr_sendmap *
3399 bbr_alloc_limit(struct tcp_bbr *bbr, uint8_t limit_type)
3400 {
3401         struct bbr_sendmap *rsm;
3402
3403         if (limit_type) {
3404                 /* currently there is only one limit type */
3405                 if (V_tcp_map_split_limit > 0 &&
3406                     bbr->r_ctl.rc_num_split_allocs >= V_tcp_map_split_limit) {
3407                         BBR_STAT_INC(bbr_split_limited);
3408                         if (!bbr->alloc_limit_reported) {
3409                                 bbr->alloc_limit_reported = 1;
3410                                 BBR_STAT_INC(bbr_alloc_limited_conns);
3411                         }
3412                         return (NULL);
3413                 }
3414         }
3415
3416         /* allocate and mark in the limit type, if set */
3417         rsm = bbr_alloc(bbr);
3418         if (rsm != NULL && limit_type) {
3419                 rsm->r_limit_type = limit_type;
3420                 bbr->r_ctl.rc_num_split_allocs++;
3421         }
3422         return (rsm);
3423 }
3424
3425 static void
3426 bbr_free(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
3427 {
3428         if (rsm->r_limit_type) {
3429                 /* currently there is only one limit type */
3430                 bbr->r_ctl.rc_num_split_allocs--;
3431         }
3432         if (rsm->r_is_smallmap)
3433                 bbr->r_ctl.rc_num_small_maps_alloced--;
3434         if (bbr->r_ctl.rc_tlp_send == rsm)
3435                 bbr->r_ctl.rc_tlp_send = NULL;
3436         if (bbr->r_ctl.rc_resend == rsm) {
3437                 bbr->r_ctl.rc_resend = NULL;
3438         }
3439         if (bbr->r_ctl.rc_next == rsm)
3440                 bbr->r_ctl.rc_next = NULL;
3441         if (bbr->r_ctl.rc_sacklast == rsm)
3442                 bbr->r_ctl.rc_sacklast = NULL;
3443         if (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
3444                 memset(rsm, 0, sizeof(struct bbr_sendmap));
3445                 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
3446                 rsm->r_limit_type = 0;
3447                 bbr->r_ctl.rc_free_cnt++;
3448                 return;
3449         }
3450         bbr->r_ctl.rc_num_maps_alloced--;
3451         uma_zfree(bbr_zone, rsm);
3452 }
3453
3454 /*
3455  * Returns the BDP.
3456  */
3457 static uint64_t
3458 bbr_get_bw_delay_prod(uint64_t rtt, uint64_t bw) {
3459         /*
3460          * Calculate the bytes in flight needed given the bw (in bytes per
3461          * second) and the specifyed rtt in useconds. We need to put out the
3462          * returned value per RTT to match that rate. Gain will normaly
3463          * raise it up from there.
3464          *
3465          * This should not overflow as long as the bandwidth is below 1
3466          * TByte per second (bw < 10**12 = 2**40) and the rtt is smaller
3467          * than 1000 seconds (rtt < 10**3 * 10**6 = 10**9 = 2**30).
3468          */
3469         uint64_t usec_per_sec;
3470
3471         usec_per_sec = USECS_IN_SECOND;
3472         return ((rtt * bw) / usec_per_sec);
3473 }
3474
3475 /*
3476  * Return the initial cwnd.
3477  */
3478 static uint32_t
3479 bbr_initial_cwnd(struct tcp_bbr *bbr, struct tcpcb *tp)
3480 {
3481         uint32_t i_cwnd;
3482
3483         if (bbr->rc_init_win) {
3484                 i_cwnd = bbr->rc_init_win * tp->t_maxseg;
3485         } else if (V_tcp_initcwnd_segments)
3486                 i_cwnd = min((V_tcp_initcwnd_segments * tp->t_maxseg),
3487                     max(2 * tp->t_maxseg, 14600));
3488         else if (V_tcp_do_rfc3390)
3489                 i_cwnd = min(4 * tp->t_maxseg,
3490                     max(2 * tp->t_maxseg, 4380));
3491         else {
3492                 /* Per RFC5681 Section 3.1 */
3493                 if (tp->t_maxseg > 2190)
3494                         i_cwnd = 2 * tp->t_maxseg;
3495                 else if (tp->t_maxseg > 1095)
3496                         i_cwnd = 3 * tp->t_maxseg;
3497                 else
3498                         i_cwnd = 4 * tp->t_maxseg;
3499         }
3500         return (i_cwnd);
3501 }
3502
3503 /*
3504  * Given a specified gain, return the target
3505  * cwnd based on that gain.
3506  */
3507 static uint32_t
3508 bbr_get_raw_target_cwnd(struct tcp_bbr *bbr, uint32_t gain, uint64_t bw)
3509 {
3510         uint64_t bdp, rtt;
3511         uint32_t cwnd;
3512
3513         if ((get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) ||
3514             (bbr_get_full_bw(bbr) == 0)) {
3515                 /* No measurements yet */
3516                 return (bbr_initial_cwnd(bbr, bbr->rc_tp));
3517         }
3518         /*
3519          * Get bytes per RTT needed (rttProp is normally in
3520          * bbr_cwndtarget_rtt_touse)
3521          */
3522         rtt = bbr_get_rtt(bbr, bbr_cwndtarget_rtt_touse);
3523         /* Get the bdp from the two values */
3524         bdp = bbr_get_bw_delay_prod(rtt, bw);
3525         /* Now apply the gain */
3526         cwnd = (uint32_t)(((bdp * ((uint64_t)gain)) + (uint64_t)(BBR_UNIT - 1)) / ((uint64_t)BBR_UNIT));
3527
3528         return (cwnd);
3529 }
3530
3531 static uint32_t
3532 bbr_get_target_cwnd(struct tcp_bbr *bbr, uint64_t bw, uint32_t gain)
3533 {
3534         uint32_t cwnd, mss;
3535
3536         mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
3537         /* Get the base cwnd with gain rounded to a mss */
3538         cwnd = roundup(bbr_get_raw_target_cwnd(bbr, bw, gain), mss);
3539         /*
3540          * Add in N (2 default since we do not have a
3541          * fq layer to trap packets in) quanta's per the I-D
3542          * section 4.2.3.2 quanta adjust.
3543          */
3544         cwnd += (bbr_quanta * bbr->r_ctl.rc_pace_max_segs);
3545         if (bbr->rc_use_google) {
3546                 if((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3547                    (bbr_state_val(bbr) == BBR_SUB_GAIN)) {
3548                         /*
3549                          * The linux implementation adds
3550                          * an extra 2 x mss in gain cycle which
3551                          * is documented no-where except in the code.
3552                          * so we add more for Neal undocumented feature
3553                          */
3554                         cwnd += 2 * mss;
3555                 }
3556                 if ((cwnd / mss) & 0x1) {
3557                         /* Round up for odd num mss */
3558                         cwnd += mss;
3559                 }
3560         }
3561         /* Are we below the min cwnd? */
3562         if (cwnd < get_min_cwnd(bbr))
3563                 return (get_min_cwnd(bbr));
3564         return (cwnd);
3565 }
3566
3567 static uint16_t
3568 bbr_gain_adjust(struct tcp_bbr *bbr, uint16_t gain)
3569 {
3570         if (gain < 1)
3571                 gain = 1;
3572         return (gain);
3573 }
3574
3575 static uint32_t
3576 bbr_get_header_oh(struct tcp_bbr *bbr)
3577 {
3578         int seg_oh;
3579
3580         seg_oh = 0;
3581         if (bbr->r_ctl.rc_inc_tcp_oh) {
3582                 /* Do we include TCP overhead? */
3583                 seg_oh = (bbr->rc_last_options + sizeof(struct tcphdr));
3584         }
3585         if (bbr->r_ctl.rc_inc_ip_oh) {
3586                 /* Do we include IP overhead? */
3587 #ifdef INET6
3588                 if (bbr->r_is_v6)
3589                         seg_oh += sizeof(struct ip6_hdr);
3590                 else
3591 #endif
3592 #ifdef INET
3593                         seg_oh += sizeof(struct ip);
3594 #endif
3595         }
3596         if (bbr->r_ctl.rc_inc_enet_oh) {
3597                 /* Do we include the ethernet overhead?  */
3598                 seg_oh += sizeof(struct ether_header);
3599         }
3600         return(seg_oh);
3601 }
3602
3603
3604 static uint32_t
3605 bbr_get_pacing_length(struct tcp_bbr *bbr, uint16_t gain, uint32_t useconds_time, uint64_t bw)
3606 {
3607         uint64_t divor, res, tim;
3608
3609         if (useconds_time == 0)
3610                 return (0);
3611         gain = bbr_gain_adjust(bbr, gain);
3612         divor = (uint64_t)USECS_IN_SECOND * (uint64_t)BBR_UNIT;
3613         tim = useconds_time;
3614         res = (tim * bw * gain) / divor;
3615         if (res == 0)
3616                 res = 1;
3617         return ((uint32_t)res);
3618 }
3619
3620 /*
3621  * Given a gain and a length return the delay in useconds that
3622  * should be used to evenly space out packets
3623  * on the connection (based on the gain factor).
3624  */
3625 static uint32_t
3626 bbr_get_pacing_delay(struct tcp_bbr *bbr, uint16_t gain, int32_t len, uint32_t cts, int nolog)
3627 {
3628         uint64_t bw, lentim, res;
3629         uint32_t usecs, srtt, over = 0;
3630         uint32_t seg_oh, num_segs, maxseg;
3631
3632         if (len == 0)
3633                 return (0);
3634
3635         maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
3636         num_segs = (len + maxseg - 1) / maxseg;
3637         if (bbr->rc_use_google == 0) {
3638                 seg_oh = bbr_get_header_oh(bbr);
3639                 len += (num_segs * seg_oh);
3640         }
3641         gain = bbr_gain_adjust(bbr, gain);
3642         bw = bbr_get_bw(bbr);
3643         if (bbr->rc_use_google) {
3644                 uint64_t cbw;
3645
3646                 /*
3647                  * Reduce the b/w by the google discount
3648                  * factor 10 = 1%.
3649                  */
3650                 cbw = bw *  (uint64_t)(1000 - bbr->r_ctl.bbr_google_discount);
3651                 cbw /= (uint64_t)1000;
3652                 /* We don't apply a discount if it results in 0 */
3653                 if (cbw > 0)
3654                         bw = cbw;
3655         }
3656         lentim = ((uint64_t)len *
3657                   (uint64_t)USECS_IN_SECOND *
3658                   (uint64_t)BBR_UNIT);
3659         res = lentim / ((uint64_t)gain * bw);
3660         if (res == 0)
3661                 res = 1;
3662         usecs = (uint32_t)res;
3663         srtt = bbr_get_rtt(bbr, BBR_SRTT);
3664         if (bbr_hptsi_max_mul && bbr_hptsi_max_div &&
3665             (bbr->rc_use_google == 0) &&
3666             (usecs > ((srtt * bbr_hptsi_max_mul) / bbr_hptsi_max_div))) {
3667                 /*
3668                  * We cannot let the delay be more than 1/2 the srtt time.
3669                  * Otherwise we cannot pace out or send properly.
3670                  */
3671                 over = usecs = (srtt * bbr_hptsi_max_mul) / bbr_hptsi_max_div;
3672                 BBR_STAT_INC(bbr_hpts_min_time);
3673         }
3674         if (!nolog)
3675                 bbr_log_pacing_delay_calc(bbr, gain, len, cts, usecs, bw, over, 1);
3676         return (usecs);
3677 }
3678
3679 static void
3680 bbr_ack_received(struct tcpcb *tp, struct tcp_bbr *bbr, struct tcphdr *th, uint32_t bytes_this_ack,
3681                  uint32_t sack_changed, uint32_t prev_acked, int32_t line, uint32_t losses)
3682 {
3683         INP_WLOCK_ASSERT(tp->t_inpcb);
3684         uint64_t bw;
3685         uint32_t cwnd, target_cwnd, saved_bytes, maxseg;
3686         int32_t meth;
3687
3688 #ifdef STATS
3689         if ((tp->t_flags & TF_GPUTINPROG) &&
3690             SEQ_GEQ(th->th_ack, tp->gput_ack)) {
3691                 /*
3692                  * Strech acks and compressed acks will cause this to
3693                  * oscillate but we are doing it the same way as the main
3694                  * stack so it will be compariable (though possibly not
3695                  * ideal).
3696                  */
3697                 int32_t cgput;
3698                 int64_t gput, time_stamp;
3699
3700                 gput = (int64_t) (th->th_ack - tp->gput_seq) * 8;
3701                 time_stamp = max(1, ((bbr->r_ctl.rc_rcvtime - tp->gput_ts) / 1000));
3702                 cgput = gput / time_stamp;
3703                 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_GPUT,
3704                                          cgput);
3705                 if (tp->t_stats_gput_prev > 0)
3706                         stats_voi_update_abs_s32(tp->t_stats,
3707                                                  VOI_TCP_GPUT_ND,
3708                                                  ((gput - tp->t_stats_gput_prev) * 100) /
3709                                                  tp->t_stats_gput_prev);
3710                 tp->t_flags &= ~TF_GPUTINPROG;
3711                 tp->t_stats_gput_prev = cgput;
3712         }
3713 #endif
3714         if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3715             ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
3716                 /* We don't change anything in probe-rtt */
3717                 return;
3718         }
3719         maxseg = tp->t_maxseg - bbr->rc_last_options;
3720         saved_bytes = bytes_this_ack;
3721         bytes_this_ack += sack_changed;
3722         if (bytes_this_ack > prev_acked) {
3723                 bytes_this_ack -= prev_acked;
3724                 /*
3725                  * A byte ack'd gives us a full mss
3726                  * to be like linux i.e. they count packets.
3727                  */
3728                 if ((bytes_this_ack < maxseg) && bbr->rc_use_google)
3729                         bytes_this_ack = maxseg;
3730         } else {
3731                 /* Unlikely */
3732                 bytes_this_ack = 0;
3733         }
3734         cwnd = tp->snd_cwnd;
3735         bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3736         if (bw)
3737                 target_cwnd = bbr_get_target_cwnd(bbr,
3738                                                   bw,
3739                                                   (uint32_t)bbr->r_ctl.rc_bbr_cwnd_gain);
3740         else
3741                 target_cwnd = bbr_initial_cwnd(bbr, bbr->rc_tp);
3742         if (IN_RECOVERY(tp->t_flags) &&
3743             (bbr->bbr_prev_in_rec == 0)) {
3744                 /*
3745                  * We are entering recovery and
3746                  * thus packet conservation.
3747                  */
3748                 bbr->pkt_conservation = 1;
3749                 bbr->r_ctl.rc_recovery_start = bbr->r_ctl.rc_rcvtime;
3750                 cwnd = ctf_flight_size(tp,
3751                                        (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
3752                         bytes_this_ack;
3753         }
3754         if (IN_RECOVERY(tp->t_flags)) {
3755                 uint32_t flight;
3756
3757                 bbr->bbr_prev_in_rec = 1;
3758                 if (cwnd > losses) {
3759                         cwnd -= losses;
3760                         if (cwnd < maxseg)
3761                                 cwnd = maxseg;
3762                 } else
3763                         cwnd = maxseg;
3764                 flight = ctf_flight_size(tp,
3765                                          (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3766                 bbr_log_type_cwndupd(bbr, flight, 0,
3767                                      losses, 10, 0, 0, line);
3768                 if (bbr->pkt_conservation) {
3769                         uint32_t time_in;
3770
3771                         if (TSTMP_GEQ(bbr->r_ctl.rc_rcvtime, bbr->r_ctl.rc_recovery_start))
3772                                 time_in = bbr->r_ctl.rc_rcvtime - bbr->r_ctl.rc_recovery_start;
3773                         else
3774                                 time_in = 0;
3775
3776                         if (time_in >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {
3777                                 /* Clear packet conservation after an rttProp */
3778                                 bbr->pkt_conservation = 0;
3779                         } else {
3780                                 if ((flight + bytes_this_ack) > cwnd)
3781                                         cwnd = flight + bytes_this_ack;
3782                                 if (cwnd < get_min_cwnd(bbr))
3783                                         cwnd = get_min_cwnd(bbr);
3784                                 tp->snd_cwnd = cwnd;
3785                                 bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed,
3786                                                      prev_acked, 1, target_cwnd, th->th_ack, line);
3787                                 return;
3788                         }
3789                 }
3790         } else
3791                 bbr->bbr_prev_in_rec = 0;
3792         if ((bbr->rc_use_google == 0) && bbr->r_ctl.restrict_growth) {
3793                 bbr->r_ctl.restrict_growth--;
3794                 if (bytes_this_ack > maxseg)
3795                         bytes_this_ack = maxseg;
3796         }
3797         if (bbr->rc_filled_pipe) {
3798                 /*
3799                  * Here we have exited startup and filled the pipe. We will
3800                  * thus allow the cwnd to shrink to the target. We hit here
3801                  * mostly.
3802                  */
3803                 uint32_t s_cwnd;
3804
3805                 meth = 2;
3806                 s_cwnd = min((cwnd + bytes_this_ack), target_cwnd);
3807                 if (s_cwnd > cwnd)
3808                         cwnd = s_cwnd;
3809                 else if (bbr_cwnd_may_shrink || bbr->rc_use_google || bbr->rc_no_pacing)
3810                         cwnd = s_cwnd;
3811         } else {
3812                 /*
3813                  * Here we are still in startup, we increase cwnd by what
3814                  * has been acked.
3815                  */
3816                 if ((cwnd < target_cwnd) ||
3817                     (bbr->rc_past_init_win == 0)) {
3818                         meth = 3;
3819                         cwnd += bytes_this_ack;
3820                 } else {
3821                         /*
3822                          * Method 4 means we are at target so no gain in
3823                          * startup and past the initial window.
3824                          */
3825                         meth = 4;
3826                 }
3827         }
3828         tp->snd_cwnd = max(cwnd, get_min_cwnd(bbr));
3829         bbr_log_type_cwndupd(bbr, saved_bytes, sack_changed, prev_acked, meth, target_cwnd, th->th_ack, line);
3830 }
3831
3832 static void
3833 tcp_bbr_partialack(struct tcpcb *tp)
3834 {
3835         struct tcp_bbr *bbr;
3836
3837         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3838         INP_WLOCK_ASSERT(tp->t_inpcb);
3839         if (ctf_flight_size(tp,
3840                 (bbr->r_ctl.rc_sacked  + bbr->r_ctl.rc_lost_bytes)) <=
3841             tp->snd_cwnd) {
3842                 bbr->r_wanted_output = 1;
3843         }
3844 }
3845
3846 static void
3847 bbr_post_recovery(struct tcpcb *tp)
3848 {
3849         struct tcp_bbr *bbr;
3850         uint32_t  flight;
3851
3852         INP_WLOCK_ASSERT(tp->t_inpcb);
3853         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
3854         /*
3855          * Here we just exit recovery.
3856          */
3857         EXIT_RECOVERY(tp->t_flags);
3858         /* Lock in our b/w reduction for the specified number of pkt-epochs */
3859         bbr->r_recovery_bw = 0;
3860         tp->snd_recover = tp->snd_una;
3861         tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3862         bbr->pkt_conservation = 0;
3863         if (bbr->rc_use_google == 0) {
3864                 /*
3865                  * For non-google mode lets
3866                  * go ahead and make sure we clear
3867                  * the recovery state so if we
3868                  * bounce back in to recovery we
3869                  * will do PC.
3870                  */
3871                 bbr->bbr_prev_in_rec = 0;
3872         }
3873         bbr_log_type_exit_rec(bbr);
3874         if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
3875                 tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
3876                 bbr_log_type_cwndupd(bbr, 0, 0, 0, 15, 0, 0, __LINE__);
3877         } else {
3878                 /* For probe-rtt case lets fix up its saved_cwnd */
3879                 if (bbr->r_ctl.rc_saved_cwnd < bbr->r_ctl.rc_cwnd_on_ent) {
3880                         bbr->r_ctl.rc_saved_cwnd = bbr->r_ctl.rc_cwnd_on_ent;
3881                         bbr_log_type_cwndupd(bbr, 0, 0, 0, 16, 0, 0, __LINE__);
3882                 }
3883         }
3884         flight = ctf_flight_size(tp,
3885                      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
3886         if ((bbr->rc_use_google == 0) &&
3887             bbr_do_red) {
3888                 uint64_t val, lr2use;
3889                 uint32_t maxseg, newcwnd, acks_inflight, ratio, cwnd;
3890                 uint32_t *cwnd_p;
3891
3892                 if (bbr_get_rtt(bbr, BBR_SRTT)) {
3893                         val = ((uint64_t)bbr_get_rtt(bbr, BBR_RTT_PROP) * (uint64_t)1000);
3894                         val /= bbr_get_rtt(bbr, BBR_SRTT);
3895                         ratio = (uint32_t)val;
3896                 } else
3897                         ratio = 1000;
3898
3899                 bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div,
3900                                      bbr->r_ctl.recovery_lr, 21,
3901                                      ratio,
3902                                      bbr->r_ctl.rc_red_cwnd_pe,
3903                                      __LINE__);
3904                 if ((ratio < bbr_do_red) || (bbr_do_red == 0))
3905                         goto done;
3906                 if (((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
3907                      bbr_prtt_slam_cwnd) ||
3908                     (bbr_sub_drain_slam_cwnd &&
3909                      (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
3910                      bbr->rc_hit_state_1 &&
3911                      (bbr_state_val(bbr) == BBR_SUB_DRAIN)) ||
3912                     ((bbr->rc_bbr_state == BBR_STATE_DRAIN) &&
3913                      bbr_slam_cwnd_in_main_drain)) {
3914                         /*
3915                          * Here we must poke at the saved cwnd
3916                          * as well as the cwnd.
3917                          */
3918                         cwnd = bbr->r_ctl.rc_saved_cwnd;
3919                         cwnd_p = &bbr->r_ctl.rc_saved_cwnd;
3920                 } else {
3921                         cwnd = tp->snd_cwnd;
3922                         cwnd_p = &tp->snd_cwnd;
3923                 }
3924                 maxseg = tp->t_maxseg - bbr->rc_last_options;
3925                 /* Add the overall lr with the recovery lr */
3926                 if (bbr->r_ctl.rc_lost == 0)
3927                         lr2use = 0;
3928                 else if (bbr->r_ctl.rc_delivered == 0)
3929                         lr2use = 1000;
3930                 else {
3931                         lr2use = bbr->r_ctl.rc_lost * 1000;
3932                         lr2use /= bbr->r_ctl.rc_delivered;
3933                 }
3934                 lr2use += bbr->r_ctl.recovery_lr;
3935                 acks_inflight = (flight / (maxseg * 2));
3936                 if (bbr_red_scale) {
3937                         lr2use *= bbr_get_rtt(bbr, BBR_SRTT);
3938                         lr2use /= bbr_red_scale;
3939                         if ((bbr_red_growth_restrict) &&
3940                             ((bbr_get_rtt(bbr, BBR_SRTT)/bbr_red_scale) > 1))
3941                             bbr->r_ctl.restrict_growth += acks_inflight;
3942                 }
3943                 if (lr2use) {
3944                         val = (uint64_t)cwnd * lr2use;
3945                         val /= 1000;
3946                         if (cwnd > val)
3947                                 newcwnd = roundup((cwnd - val), maxseg);
3948                         else
3949                                 newcwnd = maxseg;
3950                 } else {
3951                         val = (uint64_t)cwnd * (uint64_t)bbr_red_mul;
3952                         val /= (uint64_t)bbr_red_div;
3953                         newcwnd = roundup((uint32_t)val, maxseg);
3954                 }
3955                 /* with standard delayed acks how many acks can I expect? */
3956                 if (bbr_drop_limit == 0) {
3957                         /*
3958                          * Anticpate how much we will
3959                          * raise the cwnd based on the acks.
3960                          */
3961                         if ((newcwnd + (acks_inflight * maxseg)) < get_min_cwnd(bbr)) {
3962                                 /* We do enforce the min (with the acks) */
3963                                 newcwnd = (get_min_cwnd(bbr) - acks_inflight);
3964                         }
3965                 } else {
3966                         /*
3967                          * A strict drop limit of N is is inplace
3968                          */
3969                         if (newcwnd < (bbr_drop_limit * maxseg)) {
3970                                 newcwnd = bbr_drop_limit * maxseg;
3971                         }
3972                 }
3973                 /* For the next N acks do we restrict the growth */
3974                 *cwnd_p = newcwnd;
3975                 if (tp->snd_cwnd > newcwnd)
3976                         tp->snd_cwnd = newcwnd;
3977                 bbr_log_type_cwndupd(bbr, bbr_red_mul, bbr_red_div, val, 22,
3978                                      (uint32_t)lr2use,
3979                                      bbr_get_rtt(bbr, BBR_SRTT), __LINE__);
3980                 bbr->r_ctl.rc_red_cwnd_pe = bbr->r_ctl.rc_pkt_epoch;
3981         }
3982 done:
3983         bbr->r_ctl.recovery_lr = 0;
3984         if (flight <= tp->snd_cwnd) {
3985                 bbr->r_wanted_output = 1;
3986         }
3987         tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
3988 }
3989
3990 static void
3991 bbr_setup_red_bw(struct tcp_bbr *bbr, uint32_t cts)
3992 {
3993         bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
3994         /* Limit the drop in b/w to 1/2 our current filter. */
3995         if (bbr->r_ctl.red_bw > bbr->r_ctl.rc_bbr_cur_del_rate)
3996                 bbr->r_ctl.red_bw = bbr->r_ctl.rc_bbr_cur_del_rate;
3997         if (bbr->r_ctl.red_bw < (get_filter_value(&bbr->r_ctl.rc_delrate) / 2))
3998                 bbr->r_ctl.red_bw = get_filter_value(&bbr->r_ctl.rc_delrate) / 2;
3999         tcp_bbr_tso_size_check(bbr, cts);
4000 }
4001
4002 static void
4003 bbr_cong_signal(struct tcpcb *tp, struct tcphdr *th, uint32_t type, struct bbr_sendmap *rsm)
4004 {
4005         struct tcp_bbr *bbr;
4006
4007         INP_WLOCK_ASSERT(tp->t_inpcb);
4008         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
4009         switch (type) {
4010         case CC_NDUPACK:
4011                 if (!IN_RECOVERY(tp->t_flags)) {
4012                         tp->snd_recover = tp->snd_max;
4013                         /* Start a new epoch */
4014                         bbr_set_pktepoch(bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
4015                         if (bbr->rc_lt_is_sampling || bbr->rc_lt_use_bw) {
4016                                 /*
4017                                  * Move forward the lt epoch
4018                                  * so it won't count the truncated
4019                                  * epoch.
4020                                  */
4021                                 bbr->r_ctl.rc_lt_epoch++;
4022                         }
4023                         if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
4024                                 /*
4025                                  * Just like the policer detection code
4026                                  * if we are in startup we must push
4027                                  * forward the last startup epoch
4028                                  * to hide the truncated PE.
4029                                  */
4030                                 bbr->r_ctl.rc_bbr_last_startup_epoch++;
4031                         }
4032                         bbr->r_ctl.rc_cwnd_on_ent = tp->snd_cwnd;
4033                         ENTER_RECOVERY(tp->t_flags);
4034                         bbr->rc_tlp_rtx_out = 0;
4035                         bbr->r_ctl.recovery_lr = bbr->r_ctl.rc_pkt_epoch_loss_rate;
4036                         tcp_bbr_tso_size_check(bbr, bbr->r_ctl.rc_rcvtime);
4037                         if (bbr->rc_inp->inp_in_hpts &&
4038                             ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK) == 0)) {
4039                                 /*
4040                                  * When we enter recovery, we need to restart
4041                                  * any timers. This may mean we gain an agg
4042                                  * early, which will be made up for at the last
4043                                  * rxt out.
4044                                  */
4045                                 bbr->rc_timer_first = 1;
4046                                 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
4047                         }
4048                         /*
4049                          * Calculate a new cwnd based on to the current
4050                          * delivery rate with no gain. We get the bdp
4051                          * without gaining it up like we normally would and
4052                          * we use the last cur_del_rate.
4053                          */
4054                         if ((bbr->rc_use_google == 0) &&
4055                             (bbr->r_ctl.bbr_rttprobe_gain_val ||
4056                              (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT))) {
4057                                 tp->snd_cwnd = ctf_flight_size(tp,
4058                                                    (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
4059                                         (tp->t_maxseg - bbr->rc_last_options);
4060                                 if (tp->snd_cwnd < get_min_cwnd(bbr)) {
4061                                         /* We always gate to min cwnd */
4062                                         tp->snd_cwnd = get_min_cwnd(bbr);
4063                                 }
4064                                 bbr_log_type_cwndupd(bbr, 0, 0, 0, 14, 0, 0, __LINE__);
4065                         }
4066                         bbr_log_type_enter_rec(bbr, rsm->r_start);
4067                 }
4068                 break;
4069         case CC_RTO_ERR:
4070                 KMOD_TCPSTAT_INC(tcps_sndrexmitbad);
4071                 /* RTO was unnecessary, so reset everything. */
4072                 bbr_reset_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime);
4073                 if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
4074                         tp->snd_cwnd = tp->snd_cwnd_prev;
4075                         tp->snd_ssthresh = tp->snd_ssthresh_prev;
4076                         tp->snd_recover = tp->snd_recover_prev;
4077                         tp->snd_cwnd = max(tp->snd_cwnd, bbr->r_ctl.rc_cwnd_on_ent);
4078                         bbr_log_type_cwndupd(bbr, 0, 0, 0, 13, 0, 0, __LINE__);
4079                 }
4080                 tp->t_badrxtwin = 0;
4081                 break;
4082         }
4083 }
4084
4085 /*
4086  * Indicate whether this ack should be delayed.  We can delay the ack if
4087  * following conditions are met:
4088  *      - There is no delayed ack timer in progress.
4089  *      - Our last ack wasn't a 0-sized window. We never want to delay
4090  *        the ack that opens up a 0-sized window.
4091  *      - LRO wasn't used for this segment. We make sure by checking that the
4092  *        segment size is not larger than the MSS.
4093  *      - Delayed acks are enabled or this is a half-synchronized T/TCP
4094  *        connection.
4095  *      - The data being acked is less than a full segment (a stretch ack
4096  *        of more than a segment we should ack.
4097  *      - nsegs is 1 (if its more than that we received more than 1 ack).
4098  */
4099 #define DELAY_ACK(tp, bbr, nsegs)                               \
4100         (((tp->t_flags & TF_RXWIN0SENT) == 0) &&                \
4101          ((bbr->bbr_segs_rcvd + nsegs) < tp->t_delayed_ack) &&  \
4102          (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
4103
4104 /*
4105  * Return the lowest RSM in the map of
4106  * packets still in flight that is not acked.
4107  * This should normally find on the first one
4108  * since we remove packets from the send
4109  * map after they are marked ACKED.
4110  */
4111 static struct bbr_sendmap *
4112 bbr_find_lowest_rsm(struct tcp_bbr *bbr)
4113 {
4114         struct bbr_sendmap *rsm;
4115
4116         /*
4117          * Walk the time-order transmitted list looking for an rsm that is
4118          * not acked. This will be the one that was sent the longest time
4119          * ago that is still outstanding.
4120          */
4121         TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_tmap, r_tnext) {
4122                 if (rsm->r_flags & BBR_ACKED) {
4123                         continue;
4124                 }
4125                 goto finish;
4126         }
4127 finish:
4128         return (rsm);
4129 }
4130
4131 static struct bbr_sendmap *
4132 bbr_find_high_nonack(struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
4133 {
4134         struct bbr_sendmap *prsm;
4135
4136         /*
4137          * Walk the sequence order list backward until we hit and arrive at
4138          * the highest seq not acked. In theory when this is called it
4139          * should be the last segment (which it was not).
4140          */
4141         prsm = rsm;
4142         TAILQ_FOREACH_REVERSE_FROM(prsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4143                 if (prsm->r_flags & (BBR_ACKED | BBR_HAS_FIN)) {
4144                         continue;
4145                 }
4146                 return (prsm);
4147         }
4148         return (NULL);
4149 }
4150
4151 /*
4152  * Returns to the caller the number of microseconds that
4153  * the packet can be outstanding before we think we
4154  * should have had an ack returned.
4155  */
4156 static uint32_t
4157 bbr_calc_thresh_rack(struct tcp_bbr *bbr, uint32_t srtt, uint32_t cts, struct bbr_sendmap *rsm)
4158 {
4159         /*
4160          * lro is the flag we use to determine if we have seen reordering.
4161          * If it gets set we have seen reordering. The reorder logic either
4162          * works in one of two ways:
4163          *
4164          * If reorder-fade is configured, then we track the last time we saw
4165          * re-ordering occur. If we reach the point where enough time as
4166          * passed we no longer consider reordering has occuring.
4167          *
4168          * Or if reorder-face is 0, then once we see reordering we consider
4169          * the connection to alway be subject to reordering and just set lro
4170          * to 1.
4171          *
4172          * In the end if lro is non-zero we add the extra time for
4173          * reordering in.
4174          */
4175         int32_t lro;
4176         uint32_t thresh, t_rxtcur;
4177
4178         if (srtt == 0)
4179                 srtt = 1;
4180         if (bbr->r_ctl.rc_reorder_ts) {
4181                 if (bbr->r_ctl.rc_reorder_fade) {
4182                         if (SEQ_GEQ(cts, bbr->r_ctl.rc_reorder_ts)) {
4183                                 lro = cts - bbr->r_ctl.rc_reorder_ts;
4184                                 if (lro == 0) {
4185                                         /*
4186                                          * No time as passed since the last
4187                                          * reorder, mark it as reordering.
4188                                          */
4189                                         lro = 1;
4190                                 }
4191                         } else {
4192                                 /* Negative time? */
4193                                 lro = 0;
4194                         }
4195                         if (lro > bbr->r_ctl.rc_reorder_fade) {
4196                                 /* Turn off reordering seen too */
4197                                 bbr->r_ctl.rc_reorder_ts = 0;
4198                                 lro = 0;
4199                         }
4200                 } else {
4201                         /* Reodering does not fade */
4202                         lro = 1;
4203                 }
4204         } else {
4205                 lro = 0;
4206         }
4207         thresh = srtt + bbr->r_ctl.rc_pkt_delay;
4208         if (lro) {
4209                 /* It must be set, if not you get 1/4 rtt */
4210                 if (bbr->r_ctl.rc_reorder_shift)
4211                         thresh += (srtt >> bbr->r_ctl.rc_reorder_shift);
4212                 else
4213                         thresh += (srtt >> 2);
4214         } else {
4215                 thresh += 1000;
4216         }
4217         /* We don't let the rack timeout be above a RTO */
4218         if ((bbr->rc_tp)->t_srtt == 0)
4219                 t_rxtcur = BBR_INITIAL_RTO;
4220         else
4221                 t_rxtcur = TICKS_2_USEC(bbr->rc_tp->t_rxtcur);
4222         if (thresh > t_rxtcur) {
4223                 thresh = t_rxtcur;
4224         }
4225         /* And we don't want it above the RTO max either */
4226         if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4227                 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4228         }
4229         bbr_log_thresh_choice(bbr, cts, thresh, lro, srtt, rsm, BBR_TO_FRM_RACK);
4230         return (thresh);
4231 }
4232
4233 /*
4234  * Return to the caller the amount of time in mico-seconds
4235  * that should be used for the TLP timer from the last
4236  * send time of this packet.
4237  */
4238 static uint32_t
4239 bbr_calc_thresh_tlp(struct tcpcb *tp, struct tcp_bbr *bbr,
4240     struct bbr_sendmap *rsm, uint32_t srtt,
4241     uint32_t cts)
4242 {
4243         uint32_t thresh, len, maxseg, t_rxtcur;
4244         struct bbr_sendmap *prsm;
4245
4246         if (srtt == 0)
4247                 srtt = 1;
4248         if (bbr->rc_tlp_threshold)
4249                 thresh = srtt + (srtt / bbr->rc_tlp_threshold);
4250         else
4251                 thresh = (srtt * 2);
4252         maxseg = tp->t_maxseg - bbr->rc_last_options;
4253         /* Get the previous sent packet, if any  */
4254         len = rsm->r_end - rsm->r_start;
4255
4256         /* 2.1 behavior */
4257         prsm = TAILQ_PREV(rsm, bbr_head, r_tnext);
4258         if (prsm && (len <= maxseg)) {
4259                 /*
4260                  * Two packets outstanding, thresh should be (2*srtt) +
4261                  * possible inter-packet delay (if any).
4262                  */
4263                 uint32_t inter_gap = 0;
4264                 int idx, nidx;
4265
4266                 idx = rsm->r_rtr_cnt - 1;
4267                 nidx = prsm->r_rtr_cnt - 1;
4268                 if (TSTMP_GEQ(rsm->r_tim_lastsent[nidx], prsm->r_tim_lastsent[idx])) {
4269                         /* Yes it was sent later (or at the same time) */
4270                         inter_gap = rsm->r_tim_lastsent[idx] - prsm->r_tim_lastsent[nidx];
4271                 }
4272                 thresh += inter_gap;
4273         } else if (len <= maxseg) {
4274                 /*
4275                  * Possibly compensate for delayed-ack.
4276                  */
4277                 uint32_t alt_thresh;
4278
4279                 alt_thresh = srtt + (srtt / 2) + bbr_delayed_ack_time;
4280                 if (alt_thresh > thresh)
4281                         thresh = alt_thresh;
4282         }
4283         /* Not above the current  RTO */
4284         if (tp->t_srtt == 0)
4285                 t_rxtcur = BBR_INITIAL_RTO;
4286         else
4287                 t_rxtcur = TICKS_2_USEC(tp->t_rxtcur);
4288
4289         bbr_log_thresh_choice(bbr, cts, thresh, t_rxtcur, srtt, rsm, BBR_TO_FRM_TLP);
4290         /* Not above an RTO */
4291         if (thresh > t_rxtcur) {
4292                 thresh = t_rxtcur;
4293         }
4294         /* Not above a RTO max */
4295         if (thresh > (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND)) {
4296                 thresh = (((uint32_t)bbr->rc_max_rto_sec) * USECS_IN_SECOND);
4297         }
4298         /* And now apply the user TLP min */
4299         if (thresh < bbr_tlp_min) {
4300                 thresh = bbr_tlp_min;
4301         }
4302         return (thresh);
4303 }
4304
4305 /*
4306  * Return one of three RTTs to use (in microseconds).
4307  */
4308 static __inline uint32_t
4309 bbr_get_rtt(struct tcp_bbr *bbr, int32_t rtt_type)
4310 {
4311         uint32_t f_rtt;
4312         uint32_t srtt;
4313
4314         f_rtt = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
4315         if (get_filter_value_small(&bbr->r_ctl.rc_rttprop) == 0xffffffff) {
4316                 /* We have no rtt at all */
4317                 if (bbr->rc_tp->t_srtt == 0)
4318                         f_rtt = BBR_INITIAL_RTO;
4319                 else
4320                         f_rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4321                 /*
4322                  * Since we don't know how good the rtt is apply a
4323                  * delayed-ack min
4324                  */
4325                 if (f_rtt < bbr_delayed_ack_time) {
4326                         f_rtt = bbr_delayed_ack_time;
4327                 }
4328         }
4329         /* Take the filter version or last measured pkt-rtt */
4330         if (rtt_type == BBR_RTT_PROP) {
4331                 srtt = f_rtt;
4332         } else if (rtt_type == BBR_RTT_PKTRTT) {
4333                 if (bbr->r_ctl.rc_pkt_epoch_rtt) {
4334                         srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
4335                 } else {
4336                         /* No pkt rtt yet */
4337                         srtt = f_rtt;
4338                 }
4339         } else if (rtt_type == BBR_RTT_RACK) {
4340                 srtt = bbr->r_ctl.rc_last_rtt;
4341                 /* We need to add in any internal delay for our timer */
4342                 if (bbr->rc_ack_was_delayed)
4343                         srtt += bbr->r_ctl.rc_ack_hdwr_delay;
4344         } else if (rtt_type == BBR_SRTT) {
4345                 srtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
4346         } else {
4347                 /* TSNH */
4348                 srtt = f_rtt;
4349 #ifdef BBR_INVARIANTS
4350                 panic("Unknown rtt request type %d", rtt_type);
4351 #endif
4352         }
4353         return (srtt);
4354 }
4355
4356 static int
4357 bbr_is_lost(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t cts)
4358 {
4359         uint32_t thresh;
4360
4361
4362         thresh = bbr_calc_thresh_rack(bbr, bbr_get_rtt(bbr, BBR_RTT_RACK),
4363                                       cts, rsm);
4364         if ((cts - rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)]) >= thresh) {
4365                 /* It is lost (past time) */
4366                 return (1);
4367         }
4368         return (0);
4369 }
4370
4371 /*
4372  * Return a sendmap if we need to retransmit something.
4373  */
4374 static struct bbr_sendmap *
4375 bbr_check_recovery_mode(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4376 {
4377         /*
4378          * Check to see that we don't need to fall into recovery. We will
4379          * need to do so if our oldest transmit is past the time we should
4380          * have had an ack.
4381          */
4382
4383         struct bbr_sendmap *rsm;
4384         int32_t idx;
4385
4386         if (TAILQ_EMPTY(&bbr->r_ctl.rc_map)) {
4387                 /* Nothing outstanding that we know of */
4388                 return (NULL);
4389         }
4390         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
4391         if (rsm == NULL) {
4392                 /* Nothing in the transmit map */
4393                 return (NULL);
4394         }
4395         if (tp->t_flags & TF_SENTFIN) {
4396                 /* Fin restricted, don't find anything once a fin is sent */
4397                 return (NULL);
4398         }
4399         if (rsm->r_flags & BBR_ACKED) {
4400                 /*
4401                  * Ok the first one is acked (this really should not happen
4402                  * since we remove the from the tmap once they are acked)
4403                  */
4404                 rsm = bbr_find_lowest_rsm(bbr);
4405                 if (rsm == NULL)
4406                         return (NULL);
4407         }
4408         idx = rsm->r_rtr_cnt - 1;
4409         if (SEQ_LEQ(cts, rsm->r_tim_lastsent[idx])) {
4410                 /* Send timestamp is the same or less? can't be ready */
4411                 return (NULL);
4412         }
4413         /* Get our RTT time */
4414         if (bbr_is_lost(bbr, rsm, cts) &&
4415             ((rsm->r_dupack >= DUP_ACK_THRESHOLD) ||
4416              (rsm->r_flags & BBR_SACK_PASSED))) {
4417                 if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4418                         rsm->r_flags |= BBR_MARKED_LOST;
4419                         bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4420                         bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4421                 }
4422                 bbr_cong_signal(tp, NULL, CC_NDUPACK, rsm);
4423 #ifdef BBR_INVARIANTS
4424                 if ((rsm->r_end - rsm->r_start) == 0)
4425                         panic("tp:%p bbr:%p rsm:%p length is 0?", tp, bbr, rsm);
4426 #endif
4427                 return (rsm);
4428         }
4429         return (NULL);
4430 }
4431
4432 /*
4433  * RACK Timer, here we simply do logging and house keeping.
4434  * the normal bbr_output_wtime() function will call the
4435  * appropriate thing to check if we need to do a RACK retransmit.
4436  * We return 1, saying don't proceed with bbr_output_wtime only
4437  * when all timers have been stopped (destroyed PCB?).
4438  */
4439 static int
4440 bbr_timeout_rack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4441 {
4442         /*
4443          * This timer simply provides an internal trigger to send out data.
4444          * The check_recovery_mode call will see if there are needed
4445          * retransmissions, if so we will enter fast-recovery. The output
4446          * call may or may not do the same thing depending on sysctl
4447          * settings.
4448          */
4449         uint32_t lost;
4450
4451         if (bbr->rc_all_timers_stopped) {
4452                 return (1);
4453         }
4454         if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4455                 /* Its not time yet */
4456                 return (0);
4457         }
4458         BBR_STAT_INC(bbr_to_tot);
4459         lost = bbr->r_ctl.rc_lost;
4460         if (bbr->r_state && (bbr->r_state != tp->t_state))
4461                 bbr_set_state(tp, bbr, 0);
4462         bbr_log_to_event(bbr, cts, BBR_TO_FRM_RACK);
4463         if (bbr->r_ctl.rc_resend == NULL) {
4464                 /* Lets do the check here */
4465                 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
4466         }
4467         if (bbr_policer_call_from_rack_to)
4468                 bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
4469         bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RACK;
4470         return (0);
4471 }
4472
4473 static __inline void
4474 bbr_clone_rsm(struct tcp_bbr *bbr, struct bbr_sendmap *nrsm, struct bbr_sendmap *rsm, uint32_t start)
4475 {
4476         int idx;
4477
4478         nrsm->r_start = start;
4479         nrsm->r_end = rsm->r_end;
4480         nrsm->r_rtr_cnt = rsm->r_rtr_cnt;
4481         nrsm->r_flags = rsm->r_flags;
4482         /* We don't transfer forward the SYN flag */
4483         nrsm->r_flags &= ~BBR_HAS_SYN;
4484         /* We move forward the FIN flag, not that this should happen */
4485         rsm->r_flags &= ~BBR_HAS_FIN;
4486         nrsm->r_dupack = rsm->r_dupack;
4487         nrsm->r_rtr_bytes = 0;
4488         nrsm->r_is_gain = rsm->r_is_gain;
4489         nrsm->r_is_drain = rsm->r_is_drain;
4490         nrsm->r_delivered = rsm->r_delivered;
4491         nrsm->r_ts_valid = rsm->r_ts_valid;
4492         nrsm->r_del_ack_ts = rsm->r_del_ack_ts;
4493         nrsm->r_del_time = rsm->r_del_time;
4494         nrsm->r_app_limited = rsm->r_app_limited;
4495         nrsm->r_first_sent_time = rsm->r_first_sent_time;
4496         nrsm->r_flight_at_send = rsm->r_flight_at_send;
4497         /* We split a piece the lower section looses any just_ret flag. */
4498         nrsm->r_bbr_state = rsm->r_bbr_state;
4499         for (idx = 0; idx < nrsm->r_rtr_cnt; idx++) {
4500                 nrsm->r_tim_lastsent[idx] = rsm->r_tim_lastsent[idx];
4501         }
4502         rsm->r_end = nrsm->r_start;
4503         idx = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options), bbr->r_ctl.rc_pace_max_segs);
4504         idx /= 8;
4505         /* Check if we got too small */
4506         if ((rsm->r_is_smallmap == 0) &&
4507             ((rsm->r_end - rsm->r_start) <= idx)) {
4508                 bbr->r_ctl.rc_num_small_maps_alloced++;
4509                 rsm->r_is_smallmap = 1;
4510         }
4511         /* Check the new one as well */
4512         if ((nrsm->r_end - nrsm->r_start) <= idx) {
4513                 bbr->r_ctl.rc_num_small_maps_alloced++;
4514                 nrsm->r_is_smallmap = 1;
4515         }
4516 }
4517
4518 static int
4519 bbr_sack_mergable(struct bbr_sendmap *at,
4520                   uint32_t start, uint32_t end)
4521 {
4522         /*
4523          * Given a sack block defined by
4524          * start and end, and a current postion
4525          * at. Return 1 if either side of at
4526          * would show that the block is mergable
4527          * to that side. A block to be mergable
4528          * must have overlap with the start/end
4529          * and be in the SACK'd state.
4530          */
4531         struct bbr_sendmap *l_rsm;
4532         struct bbr_sendmap *r_rsm;
4533
4534         /* first get the either side blocks */
4535         l_rsm = TAILQ_PREV(at, bbr_head, r_next);
4536         r_rsm = TAILQ_NEXT(at, r_next);
4537         if (l_rsm && (l_rsm->r_flags & BBR_ACKED)) {
4538                 /* Potentially mergeable */
4539                 if ((l_rsm->r_end == start) ||
4540                     (SEQ_LT(start, l_rsm->r_end) &&
4541                      SEQ_GT(end, l_rsm->r_end))) {
4542                             /*
4543                              * map blk   |------|
4544                              * sack blk         |------|
4545                              * <or>
4546                              * map blk   |------|
4547                              * sack blk      |------|
4548                              */
4549                             return (1);
4550                     }
4551         }
4552         if (r_rsm && (r_rsm->r_flags & BBR_ACKED)) {
4553                 /* Potentially mergeable */
4554                 if ((r_rsm->r_start == end) ||
4555                     (SEQ_LT(start, r_rsm->r_start) &&
4556                      SEQ_GT(end, r_rsm->r_start))) {
4557                         /*
4558                          * map blk          |---------|
4559                          * sack blk    |----|
4560                          * <or>
4561                          * map blk          |---------|
4562                          * sack blk    |-------|
4563                          */
4564                         return (1);
4565                 }
4566         }
4567         return (0);
4568 }
4569
4570 static struct bbr_sendmap *
4571 bbr_merge_rsm(struct tcp_bbr *bbr,
4572               struct bbr_sendmap *l_rsm,
4573               struct bbr_sendmap *r_rsm)
4574 {
4575         /*
4576          * We are merging two ack'd RSM's,
4577          * the l_rsm is on the left (lower seq
4578          * values) and the r_rsm is on the right
4579          * (higher seq value). The simplest way
4580          * to merge these is to move the right
4581          * one into the left. I don't think there
4582          * is any reason we need to try to find
4583          * the oldest (or last oldest retransmitted).
4584          */
4585         l_rsm->r_end = r_rsm->r_end;
4586         if (l_rsm->r_dupack < r_rsm->r_dupack)
4587                 l_rsm->r_dupack = r_rsm->r_dupack;
4588         if (r_rsm->r_rtr_bytes)
4589                 l_rsm->r_rtr_bytes += r_rsm->r_rtr_bytes;
4590         if (r_rsm->r_in_tmap) {
4591                 /* This really should not happen */
4592                 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, r_rsm, r_tnext);
4593         }
4594         if (r_rsm->r_app_limited)
4595                 l_rsm->r_app_limited = r_rsm->r_app_limited;
4596         /* Now the flags */
4597         if (r_rsm->r_flags & BBR_HAS_FIN)
4598                 l_rsm->r_flags |= BBR_HAS_FIN;
4599         if (r_rsm->r_flags & BBR_TLP)
4600                 l_rsm->r_flags |= BBR_TLP;
4601         if (r_rsm->r_flags & BBR_RWND_COLLAPSED)
4602                 l_rsm->r_flags |= BBR_RWND_COLLAPSED;
4603         if (r_rsm->r_flags & BBR_MARKED_LOST) {
4604                 /* This really should not happen */
4605                 bbr->r_ctl.rc_lost_bytes -= r_rsm->r_end - r_rsm->r_start;
4606         }
4607         TAILQ_REMOVE(&bbr->r_ctl.rc_map, r_rsm, r_next);
4608         if ((r_rsm->r_limit_type == 0) && (l_rsm->r_limit_type != 0)) {
4609                 /* Transfer the split limit to the map we free */
4610                 r_rsm->r_limit_type = l_rsm->r_limit_type;
4611                 l_rsm->r_limit_type = 0;
4612         }
4613         bbr_free(bbr, r_rsm);
4614         return(l_rsm);
4615 }
4616
4617 /*
4618  * TLP Timer, here we simply setup what segment we want to
4619  * have the TLP expire on, the normal bbr_output_wtime() will then
4620  * send it out.
4621  *
4622  * We return 1, saying don't proceed with bbr_output_wtime only
4623  * when all timers have been stopped (destroyed PCB?).
4624  */
4625 static int
4626 bbr_timeout_tlp(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4627 {
4628         /*
4629          * Tail Loss Probe.
4630          */
4631         struct bbr_sendmap *rsm = NULL;
4632         struct socket *so;
4633         uint32_t amm;
4634         uint32_t out, avail;
4635         uint32_t maxseg;
4636         int collapsed_win = 0;
4637
4638         if (bbr->rc_all_timers_stopped) {
4639                 return (1);
4640         }
4641         if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
4642                 /* Its not time yet */
4643                 return (0);
4644         }
4645         if (bbr_progress_timeout_check(bbr)) {
4646                 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT);
4647                 return (1);
4648         }
4649         /* Did we somehow get into persists? */
4650         if (bbr->rc_in_persist) {
4651                 return (0);
4652         }
4653         if (bbr->r_state && (bbr->r_state != tp->t_state))
4654                 bbr_set_state(tp, bbr, 0);
4655         BBR_STAT_INC(bbr_tlp_tot);
4656         maxseg = tp->t_maxseg - bbr->rc_last_options;
4657 #ifdef KERN_TLS
4658         if (bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) {
4659                 /*
4660                  * For hardware TLS we do *not* want to send
4661                  * new data.
4662                  */
4663                 goto need_retran;
4664         }
4665 #endif
4666         /*
4667          * A TLP timer has expired. We have been idle for 2 rtts. So we now
4668          * need to figure out how to force a full MSS segment out.
4669          */
4670         so = tp->t_inpcb->inp_socket;
4671         avail = sbavail(&so->so_snd);
4672         out = ctf_outstanding(tp);
4673         if (out > tp->snd_wnd) {
4674                 /* special case, we need a retransmission */
4675                 collapsed_win = 1;
4676                 goto need_retran;
4677         }
4678         if (avail > out) {
4679                 /* New data is available */
4680                 amm = avail - out;
4681                 if (amm > maxseg) {
4682                         amm = maxseg;
4683                 } else if ((amm < maxseg) && ((tp->t_flags & TF_NODELAY) == 0)) {
4684                         /* not enough to fill a MTU and no-delay is off */
4685                         goto need_retran;
4686                 }
4687                 /* Set the send-new override */
4688                 if ((out + amm) <= tp->snd_wnd) {
4689                         bbr->rc_tlp_new_data = 1;
4690                 } else {
4691                         goto need_retran;
4692                 }
4693                 bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
4694                 bbr->r_ctl.rc_last_tlp_seq = tp->snd_max;
4695                 bbr->r_ctl.rc_tlp_send = NULL;
4696                 /* cap any slots */
4697                 BBR_STAT_INC(bbr_tlp_newdata);
4698                 goto send;
4699         }
4700 need_retran:
4701         /*
4702          * Ok we need to arrange the last un-acked segment to be re-sent, or
4703          * optionally the first un-acked segment.
4704          */
4705         if (collapsed_win == 0) {
4706                 rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
4707                 if (rsm && (BBR_ACKED | BBR_HAS_FIN)) {
4708                         rsm = bbr_find_high_nonack(bbr, rsm);
4709                 }
4710                 if (rsm == NULL) {
4711                         goto restore;
4712                 }
4713         } else {
4714                 /*
4715                  * We must find the last segment
4716                  * that was acceptable by the client.
4717                  */
4718                 TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
4719                         if ((rsm->r_flags & BBR_RWND_COLLAPSED) == 0) {
4720                                 /* Found one */
4721                                 break;
4722                         }
4723                 }
4724                 if (rsm == NULL) {
4725                         /* None? if so send the first */
4726                         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
4727                         if (rsm == NULL)
4728                                 goto restore;
4729                 }
4730         }
4731         if ((rsm->r_end - rsm->r_start) > maxseg) {
4732                 /*
4733                  * We need to split this the last segment in two.
4734                  */
4735                 struct bbr_sendmap *nrsm;
4736
4737                 nrsm = bbr_alloc_full_limit(bbr);
4738                 if (nrsm == NULL) {
4739                         /*
4740                          * We can't get memory to split, we can either just
4741                          * not split it. Or retransmit the whole piece, lets
4742                          * do the large send (BTLP :-) ).
4743                          */
4744                         goto go_for_it;
4745                 }
4746                 bbr_clone_rsm(bbr, nrsm, rsm, (rsm->r_end - maxseg));
4747                 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
4748                 if (rsm->r_in_tmap) {
4749                         TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
4750                         nrsm->r_in_tmap = 1;
4751                 }
4752                 rsm->r_flags &= (~BBR_HAS_FIN);
4753                 rsm = nrsm;
4754         }
4755 go_for_it:
4756         bbr->r_ctl.rc_tlp_send = rsm;
4757         bbr->rc_tlp_rtx_out = 1;
4758         if (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) {
4759                 bbr->r_ctl.rc_tlp_seg_send_cnt++;
4760                 tp->t_rxtshift++;
4761         } else {
4762                 bbr->r_ctl.rc_last_tlp_seq = rsm->r_start;
4763                 bbr->r_ctl.rc_tlp_seg_send_cnt = 1;
4764         }
4765 send:
4766         if (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend) {
4767                 /*
4768                  * Can't [re]/transmit a segment we have retranmitted the
4769                  * max times. We need the retransmit timer to take over.
4770                  */
4771 restore:
4772                 bbr->rc_tlp_new_data = 0;
4773                 bbr->r_ctl.rc_tlp_send = NULL;
4774                 if (rsm)
4775                         rsm->r_flags &= ~BBR_TLP;
4776                 BBR_STAT_INC(bbr_tlp_retran_fail);
4777                 return (0);
4778         } else if (rsm) {
4779                 rsm->r_flags |= BBR_TLP;
4780         }
4781         if (rsm && (rsm->r_start == bbr->r_ctl.rc_last_tlp_seq) &&
4782             (bbr->r_ctl.rc_tlp_seg_send_cnt > bbr_tlp_max_resend)) {
4783                 /*
4784                  * We have retransmitted to many times for TLP. Switch to
4785                  * the regular RTO timer
4786                  */
4787                 goto restore;
4788         }
4789         bbr_log_to_event(bbr, cts, BBR_TO_FRM_TLP);
4790         bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_TLP;
4791         return (0);
4792 }
4793
4794 /*
4795  * Delayed ack Timer, here we simply need to setup the
4796  * ACK_NOW flag and remove the DELACK flag. From there
4797  * the output routine will send the ack out.
4798  *
4799  * We only return 1, saying don't proceed, if all timers
4800  * are stopped (destroyed PCB?).
4801  */
4802 static int
4803 bbr_timeout_delack(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4804 {
4805         if (bbr->rc_all_timers_stopped) {
4806                 return (1);
4807         }
4808         bbr_log_to_event(bbr, cts, BBR_TO_FRM_DELACK);
4809         tp->t_flags &= ~TF_DELACK;
4810         tp->t_flags |= TF_ACKNOW;
4811         KMOD_TCPSTAT_INC(tcps_delack);
4812         bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_DELACK;
4813         return (0);
4814 }
4815
4816 /*
4817  * Persists timer, here we simply need to setup the
4818  * FORCE-DATA flag the output routine will send
4819  * the one byte send.
4820  *
4821  * We only return 1, saying don't proceed, if all timers
4822  * are stopped (destroyed PCB?).
4823  */
4824 static int
4825 bbr_timeout_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4826 {
4827         struct tcptemp *t_template;
4828         int32_t retval = 1;
4829
4830         if (bbr->rc_all_timers_stopped) {
4831                 return (1);
4832         }
4833         if (bbr->rc_in_persist == 0)
4834                 return (0);
4835         KASSERT(tp->t_inpcb != NULL,
4836             ("%s: tp %p tp->t_inpcb == NULL", __func__, tp));
4837         /*
4838          * Persistence timer into zero window. Force a byte to be output, if
4839          * possible.
4840          */
4841         bbr_log_to_event(bbr, cts, BBR_TO_FRM_PERSIST);
4842         bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_PERSIT;
4843         KMOD_TCPSTAT_INC(tcps_persisttimeo);
4844         /*
4845          * Have we exceeded the user specified progress time?
4846          */
4847         if (bbr_progress_timeout_check(bbr)) {
4848                 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT);
4849                 goto out;
4850         }
4851         /*
4852          * Hack: if the peer is dead/unreachable, we do not time out if the
4853          * window is closed.  After a full backoff, drop the connection if
4854          * the idle time (no responses to probes) reaches the maximum
4855          * backoff that we would use if retransmitting.
4856          */
4857         if (tp->t_rxtshift == TCP_MAXRXTSHIFT &&
4858             (ticks - tp->t_rcvtime >= tcp_maxpersistidle ||
4859             ticks - tp->t_rcvtime >= TCP_REXMTVAL(tp) * tcp_totbackoff)) {
4860                 KMOD_TCPSTAT_INC(tcps_persistdrop);
4861                 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT);
4862                 goto out;
4863         }
4864         if ((sbavail(&bbr->rc_inp->inp_socket->so_snd) == 0) &&
4865             tp->snd_una == tp->snd_max) {
4866                 bbr_exit_persist(tp, bbr, cts, __LINE__);
4867                 retval = 0;
4868                 goto out;
4869         }
4870         /*
4871          * If the user has closed the socket then drop a persisting
4872          * connection after a much reduced timeout.
4873          */
4874         if (tp->t_state > TCPS_CLOSE_WAIT &&
4875             (ticks - tp->t_rcvtime) >= TCPTV_PERSMAX) {
4876                 KMOD_TCPSTAT_INC(tcps_persistdrop);
4877                 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT);
4878                 goto out;
4879         }
4880         t_template = tcpip_maketemplate(bbr->rc_inp);
4881         if (t_template) {
4882                 tcp_respond(tp, t_template->tt_ipgen,
4883                             &t_template->tt_t, (struct mbuf *)NULL,
4884                             tp->rcv_nxt, tp->snd_una - 1, 0);
4885                 /* This sends an ack */
4886                 if (tp->t_flags & TF_DELACK)
4887                         tp->t_flags &= ~TF_DELACK;
4888                 free(t_template, M_TEMP);
4889         }
4890         if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
4891                 tp->t_rxtshift++;
4892         bbr_start_hpts_timer(bbr, tp, cts, 3, 0, 0);
4893 out:
4894         return (retval);
4895 }
4896
4897 /*
4898  * If a keepalive goes off, we had no other timers
4899  * happening. We always return 1 here since this
4900  * routine either drops the connection or sends
4901  * out a segment with respond.
4902  */
4903 static int
4904 bbr_timeout_keepalive(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
4905 {
4906         struct tcptemp *t_template;
4907         struct inpcb *inp;
4908
4909         if (bbr->rc_all_timers_stopped) {
4910                 return (1);
4911         }
4912         bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_KEEP;
4913         inp = tp->t_inpcb;
4914         bbr_log_to_event(bbr, cts, BBR_TO_FRM_KEEP);
4915         /*
4916          * Keep-alive timer went off; send something or drop connection if
4917          * idle for too long.
4918          */
4919         KMOD_TCPSTAT_INC(tcps_keeptimeo);
4920         if (tp->t_state < TCPS_ESTABLISHED)
4921                 goto dropit;
4922         if ((V_tcp_always_keepalive || inp->inp_socket->so_options & SO_KEEPALIVE) &&
4923             tp->t_state <= TCPS_CLOSING) {
4924                 if (ticks - tp->t_rcvtime >= TP_KEEPIDLE(tp) + TP_MAXIDLE(tp))
4925                         goto dropit;
4926                 /*
4927                  * Send a packet designed to force a response if the peer is
4928                  * up and reachable: either an ACK if the connection is
4929                  * still alive, or an RST if the peer has closed the
4930                  * connection due to timeout or reboot. Using sequence
4931                  * number tp->snd_una-1 causes the transmitted zero-length
4932                  * segment to lie outside the receive window; by the
4933                  * protocol spec, this requires the correspondent TCP to
4934                  * respond.
4935                  */
4936                 KMOD_TCPSTAT_INC(tcps_keepprobe);
4937                 t_template = tcpip_maketemplate(inp);
4938                 if (t_template) {
4939                         tcp_respond(tp, t_template->tt_ipgen,
4940                             &t_template->tt_t, (struct mbuf *)NULL,
4941                             tp->rcv_nxt, tp->snd_una - 1, 0);
4942                         free(t_template, M_TEMP);
4943                 }
4944         }
4945         bbr_start_hpts_timer(bbr, tp, cts, 4, 0, 0);
4946         return (1);
4947 dropit:
4948         KMOD_TCPSTAT_INC(tcps_keepdrops);
4949         tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT);
4950         return (1);
4951 }
4952
4953 /*
4954  * Retransmit helper function, clear up all the ack
4955  * flags and take care of important book keeping.
4956  */
4957 static void
4958 bbr_remxt_tmr(struct tcpcb *tp)
4959 {
4960         /*
4961          * The retransmit timer went off, all sack'd blocks must be
4962          * un-acked.
4963          */
4964         struct bbr_sendmap *rsm, *trsm = NULL;
4965         struct tcp_bbr *bbr;
4966         uint32_t cts, lost;
4967
4968         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
4969         cts = tcp_get_usecs(&bbr->rc_tv);
4970         lost = bbr->r_ctl.rc_lost;
4971         if (bbr->r_state && (bbr->r_state != tp->t_state))
4972                 bbr_set_state(tp, bbr, 0);
4973
4974         TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
4975                 if (rsm->r_flags & BBR_ACKED) {
4976                         uint32_t old_flags;
4977
4978                         rsm->r_dupack = 0;
4979                         if (rsm->r_in_tmap == 0) {
4980                                 /* We must re-add it back to the tlist */
4981                                 if (trsm == NULL) {
4982                                         TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
4983                                 } else {
4984                                         TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, trsm, rsm, r_tnext);
4985                                 }
4986                                 rsm->r_in_tmap = 1;
4987                         }
4988                         old_flags = rsm->r_flags;
4989                         rsm->r_flags |= BBR_RXT_CLEARED;
4990                         rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS);
4991                         bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
4992                 } else {
4993                         if ((rsm->r_flags & BBR_MARKED_LOST) == 0) {
4994                                 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
4995                                 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
4996                         }
4997                         if (bbr_marks_rxt_sack_passed) {
4998                                 /*
4999                                  * With this option, we will rack out
5000                                  * in 1ms increments the rest of the packets.
5001                                  */
5002                                 rsm->r_flags |= BBR_SACK_PASSED | BBR_MARKED_LOST;
5003                                 rsm->r_flags &= ~BBR_WAS_SACKPASS;
5004                         } else {
5005                                 /*
5006                                  * With this option we only mark them lost
5007                                  * and remove all sack'd markings. We will run
5008                                  * another RXT or a TLP. This will cause
5009                                  * us to eventually send more based on what
5010                                  * ack's come in.
5011                                  */
5012                                 rsm->r_flags |= BBR_MARKED_LOST;
5013                                 rsm->r_flags &= ~BBR_WAS_SACKPASS;
5014                                 rsm->r_flags &= ~BBR_SACK_PASSED;
5015                         }
5016                 }
5017                 trsm = rsm;
5018         }
5019         bbr->r_ctl.rc_resend = TAILQ_FIRST(&bbr->r_ctl.rc_map);
5020         /* Clear the count (we just un-acked them) */
5021         bbr_log_to_event(bbr, cts, BBR_TO_FRM_TMR);
5022         bbr->rc_tlp_new_data = 0;
5023         bbr->r_ctl.rc_tlp_seg_send_cnt = 0;
5024         /* zap the behindness on a rxt */
5025         bbr->r_ctl.rc_hptsi_agg_delay = 0;
5026         bbr->r_agg_early_set = 0;
5027         bbr->r_ctl.rc_agg_early = 0;
5028         bbr->rc_tlp_rtx_out = 0;
5029         bbr->r_ctl.rc_sacked = 0;
5030         bbr->r_ctl.rc_sacklast = NULL;
5031         bbr->r_timer_override = 1;
5032         bbr_lt_bw_sampling(bbr, cts, (bbr->r_ctl.rc_lost > lost));
5033 }
5034
5035 /*
5036  * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
5037  * we will setup to retransmit the lowest seq number outstanding.
5038  */
5039 static int
5040 bbr_timeout_rxt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
5041 {
5042         int32_t rexmt;
5043         int32_t retval = 0;
5044         bool isipv6;
5045
5046         bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_RXT;
5047         if (bbr->rc_all_timers_stopped) {
5048                 return (1);
5049         }
5050         if (TCPS_HAVEESTABLISHED(tp->t_state) &&
5051             (tp->snd_una == tp->snd_max)) {
5052                 /* Nothing outstanding .. nothing to do */
5053                 return (0);
5054         }
5055         /*
5056          * Retransmission timer went off.  Message has not been acked within
5057          * retransmit interval.  Back off to a longer retransmit interval
5058          * and retransmit one segment.
5059          */
5060         if (bbr_progress_timeout_check(bbr)) {
5061                 retval = 1;
5062                 tcp_set_inp_to_drop(bbr->rc_inp, ETIMEDOUT);
5063                 goto out;
5064         }
5065         bbr_remxt_tmr(tp);
5066         if ((bbr->r_ctl.rc_resend == NULL) ||
5067             ((bbr->r_ctl.rc_resend->r_flags & BBR_RWND_COLLAPSED) == 0)) {
5068                 /*
5069                  * If the rwnd collapsed on
5070                  * the one we are retransmitting
5071                  * it does not count against the
5072                  * rxt count.
5073                  */
5074                 tp->t_rxtshift++;
5075         }
5076         if (tp->t_rxtshift > TCP_MAXRXTSHIFT) {
5077                 tp->t_rxtshift = TCP_MAXRXTSHIFT;
5078                 KMOD_TCPSTAT_INC(tcps_timeoutdrop);
5079                 retval = 1;
5080                 tcp_set_inp_to_drop(bbr->rc_inp,
5081                     (tp->t_softerror ? (uint16_t) tp->t_softerror : ETIMEDOUT));
5082                 goto out;
5083         }
5084         if (tp->t_state == TCPS_SYN_SENT) {
5085                 /*
5086                  * If the SYN was retransmitted, indicate CWND to be limited
5087                  * to 1 segment in cc_conn_init().
5088                  */
5089                 tp->snd_cwnd = 1;
5090         } else if (tp->t_rxtshift == 1) {
5091                 /*
5092                  * first retransmit; record ssthresh and cwnd so they can be
5093                  * recovered if this turns out to be a "bad" retransmit. A
5094                  * retransmit is considered "bad" if an ACK for this segment
5095                  * is received within RTT/2 interval; the assumption here is
5096                  * that the ACK was already in flight.  See "On Estimating
5097                  * End-to-End Network Path Properties" by Allman and Paxson
5098                  * for more details.
5099                  */
5100                 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5101                 if (!IN_RECOVERY(tp->t_flags)) {
5102                         tp->snd_cwnd_prev = tp->snd_cwnd;
5103                         tp->snd_ssthresh_prev = tp->snd_ssthresh;
5104                         tp->snd_recover_prev = tp->snd_recover;
5105                         tp->t_badrxtwin = ticks + (tp->t_srtt >> (TCP_RTT_SHIFT + 1));
5106                         tp->t_flags |= TF_PREVVALID;
5107                 } else {
5108                         tp->t_flags &= ~TF_PREVVALID;
5109                 }
5110                 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5111         } else {
5112                 tp->snd_cwnd = tp->t_maxseg - bbr->rc_last_options;
5113                 tp->t_flags &= ~TF_PREVVALID;
5114         }
5115         KMOD_TCPSTAT_INC(tcps_rexmttimeo);
5116         if ((tp->t_state == TCPS_SYN_SENT) ||
5117             (tp->t_state == TCPS_SYN_RECEIVED))
5118                 rexmt = USEC_2_TICKS(BBR_INITIAL_RTO) * tcp_backoff[tp->t_rxtshift];
5119         else
5120                 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
5121         TCPT_RANGESET(tp->t_rxtcur, rexmt,
5122             MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms),
5123             MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
5124         /*
5125          * We enter the path for PLMTUD if connection is established or, if
5126          * connection is FIN_WAIT_1 status, reason for the last is that if
5127          * amount of data we send is very small, we could send it in couple
5128          * of packets and process straight to FIN. In that case we won't
5129          * catch ESTABLISHED state.
5130          */
5131 #ifdef INET6
5132         isipv6 = (tp->t_inpcb->inp_vflag & INP_IPV6) ? true : false;
5133 #else
5134         isipv6 = false;
5135 #endif
5136         if (((V_tcp_pmtud_blackhole_detect == 1) ||
5137             (V_tcp_pmtud_blackhole_detect == 2 && !isipv6) ||
5138             (V_tcp_pmtud_blackhole_detect == 3 && isipv6)) &&
5139             ((tp->t_state == TCPS_ESTABLISHED) ||
5140             (tp->t_state == TCPS_FIN_WAIT_1))) {
5141
5142                 /*
5143                  * Idea here is that at each stage of mtu probe (usually,
5144                  * 1448 -> 1188 -> 524) should be given 2 chances to recover
5145                  * before further clamping down. 'tp->t_rxtshift % 2 == 0'
5146                  * should take care of that.
5147                  */
5148                 if (((tp->t_flags2 & (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) ==
5149                     (TF2_PLPMTU_PMTUD | TF2_PLPMTU_MAXSEGSNT)) &&
5150                     (tp->t_rxtshift >= 2 && tp->t_rxtshift < 6 &&
5151                     tp->t_rxtshift % 2 == 0)) {
5152                         /*
5153                          * Enter Path MTU Black-hole Detection mechanism: -
5154                          * Disable Path MTU Discovery (IP "DF" bit). -
5155                          * Reduce MTU to lower value than what we negotiated
5156                          * with peer.
5157                          */
5158                         if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) == 0) {
5159                                 /*
5160                                  * Record that we may have found a black
5161                                  * hole.
5162                                  */
5163                                 tp->t_flags2 |= TF2_PLPMTU_BLACKHOLE;
5164                                 /* Keep track of previous MSS. */
5165                                 tp->t_pmtud_saved_maxseg = tp->t_maxseg;
5166                         }
5167                         /*
5168                          * Reduce the MSS to blackhole value or to the
5169                          * default in an attempt to retransmit.
5170                          */
5171 #ifdef INET6
5172                         isipv6 = bbr->r_is_v6;
5173                         if (isipv6 &&
5174                             tp->t_maxseg > V_tcp_v6pmtud_blackhole_mss) {
5175                                 /* Use the sysctl tuneable blackhole MSS. */
5176                                 tp->t_maxseg = V_tcp_v6pmtud_blackhole_mss;
5177                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
5178                         } else if (isipv6) {
5179                                 /* Use the default MSS. */
5180                                 tp->t_maxseg = V_tcp_v6mssdflt;
5181                                 /*
5182                                  * Disable Path MTU Discovery when we switch
5183                                  * to minmss.
5184                                  */
5185                                 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
5186                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
5187                         }
5188 #endif
5189 #if defined(INET6) && defined(INET)
5190                         else
5191 #endif
5192 #ifdef INET
5193                         if (tp->t_maxseg > V_tcp_pmtud_blackhole_mss) {
5194                                 /* Use the sysctl tuneable blackhole MSS. */
5195                                 tp->t_maxseg = V_tcp_pmtud_blackhole_mss;
5196                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated);
5197                         } else {
5198                                 /* Use the default MSS. */
5199                                 tp->t_maxseg = V_tcp_mssdflt;
5200                                 /*
5201                                  * Disable Path MTU Discovery when we switch
5202                                  * to minmss.
5203                                  */
5204                                 tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
5205                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_activated_min_mss);
5206                         }
5207 #endif
5208                 } else {
5209                         /*
5210                          * If further retransmissions are still unsuccessful
5211                          * with a lowered MTU, maybe this isn't a blackhole
5212                          * and we restore the previous MSS and blackhole
5213                          * detection flags. The limit '6' is determined by
5214                          * giving each probe stage (1448, 1188, 524) 2
5215                          * chances to recover.
5216                          */
5217                         if ((tp->t_flags2 & TF2_PLPMTU_BLACKHOLE) &&
5218                             (tp->t_rxtshift >= 6)) {
5219                                 tp->t_flags2 |= TF2_PLPMTU_PMTUD;
5220                                 tp->t_flags2 &= ~TF2_PLPMTU_BLACKHOLE;
5221                                 tp->t_maxseg = tp->t_pmtud_saved_maxseg;
5222                                 KMOD_TCPSTAT_INC(tcps_pmtud_blackhole_failed);
5223                         }
5224                 }
5225         }
5226         /*
5227          * Disable RFC1323 and SACK if we haven't got any response to our
5228          * third SYN to work-around some broken terminal servers (most of
5229          * which have hopefully been retired) that have bad VJ header
5230          * compression code which trashes TCP segments containing
5231          * unknown-to-them TCP options.
5232          */
5233         if (tcp_rexmit_drop_options && (tp->t_state == TCPS_SYN_SENT) &&
5234             (tp->t_rxtshift == 3))
5235                 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_SACK_PERMIT);
5236         /*
5237          * If we backed off this far, our srtt estimate is probably bogus.
5238          * Clobber it so we'll take the next rtt measurement as our srtt;
5239          * move the current srtt into rttvar to keep the current retransmit
5240          * times until then.
5241          */
5242         if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
5243 #ifdef INET6
5244                 if (bbr->r_is_v6)
5245                         in6_losing(tp->t_inpcb);
5246                 else
5247 #endif
5248                         in_losing(tp->t_inpcb);
5249                 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
5250                 tp->t_srtt = 0;
5251         }
5252         sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
5253         tp->snd_recover = tp->snd_max;
5254         tp->t_flags |= TF_ACKNOW;
5255         tp->t_rtttime = 0;
5256 out:
5257         return (retval);
5258 }
5259
5260 static int
5261 bbr_process_timers(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, uint8_t hpts_calling)
5262 {
5263         int32_t ret = 0;
5264         int32_t timers = (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK);
5265
5266         if (timers == 0) {
5267                 return (0);
5268         }
5269         if (tp->t_state == TCPS_LISTEN) {
5270                 /* no timers on listen sockets */
5271                 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)
5272                         return (0);
5273                 return (1);
5274         }
5275         if (TSTMP_LT(cts, bbr->r_ctl.rc_timer_exp)) {
5276                 uint32_t left;
5277
5278                 if (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) {
5279                         ret = -1;
5280                         bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling);
5281                         return (0);
5282                 }
5283                 if (hpts_calling == 0) {
5284                         ret = -2;
5285                         bbr_log_to_processing(bbr, cts, ret, 0, hpts_calling);
5286                         return (0);
5287                 }
5288                 /*
5289                  * Ok our timer went off early and we are not paced false
5290                  * alarm, go back to sleep.
5291                  */
5292                 left = bbr->r_ctl.rc_timer_exp - cts;
5293                 ret = -3;
5294                 bbr_log_to_processing(bbr, cts, ret, left, hpts_calling);
5295                 tcp_hpts_insert(tp->t_inpcb, HPTS_USEC_TO_SLOTS(left));
5296                 return (1);
5297         }
5298         bbr->rc_tmr_stopped = 0;
5299         bbr->r_ctl.rc_hpts_flags &= ~PACE_TMR_MASK;
5300         if (timers & PACE_TMR_DELACK) {
5301                 ret = bbr_timeout_delack(tp, bbr, cts);
5302         } else if (timers & PACE_TMR_PERSIT) {
5303                 ret = bbr_timeout_persist(tp, bbr, cts);
5304         } else if (timers & PACE_TMR_RACK) {
5305                 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5306                 ret = bbr_timeout_rack(tp, bbr, cts);
5307         } else if (timers & PACE_TMR_TLP) {
5308                 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5309                 ret = bbr_timeout_tlp(tp, bbr, cts);
5310         } else if (timers & PACE_TMR_RXT) {
5311                 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
5312                 ret = bbr_timeout_rxt(tp, bbr, cts);
5313         } else if (timers & PACE_TMR_KEEP) {
5314                 ret = bbr_timeout_keepalive(tp, bbr, cts);
5315         }
5316         bbr_log_to_processing(bbr, cts, ret, timers, hpts_calling);
5317         return (ret);
5318 }
5319
5320 static void
5321 bbr_timer_cancel(struct tcp_bbr *bbr, int32_t line, uint32_t cts)
5322 {
5323         if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
5324                 uint8_t hpts_removed = 0;
5325
5326                 if (bbr->rc_inp->inp_in_hpts &&
5327                     (bbr->rc_timer_first == 1)) {
5328                         /*
5329                          * If we are canceling timer's when we have the
5330                          * timer ahead of the output being paced. We also
5331                          * must remove ourselves from the hpts.
5332                          */
5333                         hpts_removed = 1;
5334                         tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
5335                         if (bbr->r_ctl.rc_last_delay_val) {
5336                                 /* Update the last hptsi delay too */
5337                                 uint32_t time_since_send;
5338
5339                                 if (TSTMP_GT(cts, bbr->rc_pacer_started))
5340                                         time_since_send = cts - bbr->rc_pacer_started;
5341                                 else
5342                                         time_since_send = 0;
5343                                 if (bbr->r_ctl.rc_last_delay_val > time_since_send) {
5344                                         /* Cut down our slot time */
5345                                         bbr->r_ctl.rc_last_delay_val -= time_since_send;
5346                                 } else {
5347                                         bbr->r_ctl.rc_last_delay_val = 0;
5348                                 }
5349                                 bbr->rc_pacer_started = cts;
5350                         }
5351                 }
5352                 bbr->rc_timer_first = 0;
5353                 bbr_log_to_cancel(bbr, line, cts, hpts_removed);
5354                 bbr->rc_tmr_stopped = bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK;
5355                 bbr->r_ctl.rc_hpts_flags &= ~(PACE_TMR_MASK);
5356         }
5357 }
5358
5359 static void
5360 bbr_timer_stop(struct tcpcb *tp, uint32_t timer_type)
5361 {
5362         struct tcp_bbr *bbr;
5363
5364         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
5365         bbr->rc_all_timers_stopped = 1;
5366         return;
5367 }
5368
5369 /*
5370  * stop all timers always returning 0.
5371  */
5372 static int
5373 bbr_stopall(struct tcpcb *tp)
5374 {
5375         return (0);
5376 }
5377
5378 static void
5379 bbr_timer_activate(struct tcpcb *tp, uint32_t timer_type, uint32_t delta)
5380 {
5381         return;
5382 }
5383
5384 /*
5385  * return true if a bbr timer (rack or tlp) is active.
5386  */
5387 static int
5388 bbr_timer_active(struct tcpcb *tp, uint32_t timer_type)
5389 {
5390         return (0);
5391 }
5392
5393 static uint32_t
5394 bbr_get_earliest_send_outstanding(struct tcp_bbr *bbr, struct bbr_sendmap *u_rsm, uint32_t cts)
5395 {
5396         struct bbr_sendmap *rsm;
5397
5398         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
5399         if ((rsm == NULL) || (u_rsm == rsm))
5400                 return (cts);
5401         return(rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]);
5402 }
5403
5404 static void
5405 bbr_update_rsm(struct tcpcb *tp, struct tcp_bbr *bbr,
5406      struct bbr_sendmap *rsm, uint32_t cts, uint32_t pacing_time)
5407 {
5408         int32_t idx;
5409
5410         rsm->r_rtr_cnt++;
5411         rsm->r_dupack = 0;
5412         if (rsm->r_rtr_cnt > BBR_NUM_OF_RETRANS) {
5413                 rsm->r_rtr_cnt = BBR_NUM_OF_RETRANS;
5414                 rsm->r_flags |= BBR_OVERMAX;
5415         }
5416         if (rsm->r_flags & BBR_RWND_COLLAPSED) {
5417                 /* Take off the collapsed flag at rxt */
5418                 rsm->r_flags &= ~BBR_RWND_COLLAPSED;
5419         }
5420         if (rsm->r_flags & BBR_MARKED_LOST) {
5421                 /* We have retransmitted, its no longer lost */
5422                 rsm->r_flags &= ~BBR_MARKED_LOST;
5423                 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
5424         }
5425         if (rsm->r_flags & BBR_RXT_CLEARED) {
5426                 /*
5427                  * We hit a RXT timer on it and
5428                  * we cleared the "acked" flag.
5429                  * We now have it going back into
5430                  * flight, we can remove the cleared
5431                  * flag and possibly do accounting on
5432                  * this piece.
5433                  */
5434                 rsm->r_flags &= ~BBR_RXT_CLEARED;
5435         }
5436         if ((rsm->r_rtr_cnt > 1) && ((rsm->r_flags & BBR_TLP) == 0)) {
5437                 bbr->r_ctl.rc_holes_rxt += (rsm->r_end - rsm->r_start);
5438                 rsm->r_rtr_bytes += (rsm->r_end - rsm->r_start);
5439         }
5440         idx = rsm->r_rtr_cnt - 1;
5441         rsm->r_tim_lastsent[idx] = cts;
5442         rsm->r_pacing_delay = pacing_time;
5443         rsm->r_delivered = bbr->r_ctl.rc_delivered;
5444         rsm->r_ts_valid = bbr->rc_ts_valid;
5445         if (bbr->rc_ts_valid)
5446                 rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
5447         if (bbr->r_ctl.r_app_limited_until)
5448                 rsm->r_app_limited = 1;
5449         else
5450                 rsm->r_app_limited = 0;
5451         if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
5452                 rsm->r_bbr_state = bbr_state_val(bbr);
5453         else
5454                 rsm->r_bbr_state = 8;
5455         if (rsm->r_flags & BBR_ACKED) {
5456                 /* Problably MTU discovery messing with us */
5457                 uint32_t old_flags;
5458
5459                 old_flags = rsm->r_flags;
5460                 rsm->r_flags &= ~BBR_ACKED;
5461                 bbr_log_type_rsmclear(bbr, cts, rsm, old_flags, __LINE__);
5462                 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
5463                 if (bbr->r_ctl.rc_sacked == 0)
5464                         bbr->r_ctl.rc_sacklast = NULL;
5465         }
5466         if (rsm->r_in_tmap) {
5467                 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5468         }
5469         TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
5470         rsm->r_in_tmap = 1;
5471         if (rsm->r_flags & BBR_SACK_PASSED) {
5472                 /* We have retransmitted due to the SACK pass */
5473                 rsm->r_flags &= ~BBR_SACK_PASSED;
5474                 rsm->r_flags |= BBR_WAS_SACKPASS;
5475         }
5476         rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
5477         rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
5478                                                 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
5479         bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
5480         if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
5481                 rsm->r_is_gain = 1;
5482                 rsm->r_is_drain = 0;
5483         } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
5484                 rsm->r_is_drain = 1;
5485                 rsm->r_is_gain = 0;
5486         } else {
5487                 rsm->r_is_drain = 0;
5488                 rsm->r_is_gain = 0;
5489         }
5490         rsm->r_del_time = bbr->r_ctl.rc_del_time; /* TEMP GOOGLE CODE */
5491 }
5492
5493 /*
5494  * Returns 0, or the sequence where we stopped
5495  * updating. We also update the lenp to be the amount
5496  * of data left.
5497  */
5498
5499 static uint32_t
5500 bbr_update_entry(struct tcpcb *tp, struct tcp_bbr *bbr,
5501     struct bbr_sendmap *rsm, uint32_t cts, int32_t *lenp, uint32_t pacing_time)
5502 {
5503         /*
5504          * We (re-)transmitted starting at rsm->r_start for some length
5505          * (possibly less than r_end.
5506          */
5507         struct bbr_sendmap *nrsm;
5508         uint32_t c_end;
5509         int32_t len;
5510
5511         len = *lenp;
5512         c_end = rsm->r_start + len;
5513         if (SEQ_GEQ(c_end, rsm->r_end)) {
5514                 /*
5515                  * We retransmitted the whole piece or more than the whole
5516                  * slopping into the next rsm.
5517                  */
5518                 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5519                 if (c_end == rsm->r_end) {
5520                         *lenp = 0;
5521                         return (0);
5522                 } else {
5523                         int32_t act_len;
5524
5525                         /* Hangs over the end return whats left */
5526                         act_len = rsm->r_end - rsm->r_start;
5527                         *lenp = (len - act_len);
5528                         return (rsm->r_end);
5529                 }
5530                 /* We don't get out of this block. */
5531         }
5532         /*
5533          * Here we retransmitted less than the whole thing which means we
5534          * have to split this into what was transmitted and what was not.
5535          */
5536         nrsm = bbr_alloc_full_limit(bbr);
5537         if (nrsm == NULL) {
5538                 *lenp = 0;
5539                 return (0);
5540         }
5541         /*
5542          * So here we are going to take the original rsm and make it what we
5543          * retransmitted. nrsm will be the tail portion we did not
5544          * retransmit. For example say the chunk was 1, 11 (10 bytes). And
5545          * we retransmitted 5 bytes i.e. 1, 5. The original piece shrinks to
5546          * 1, 6 and the new piece will be 6, 11.
5547          */
5548         bbr_clone_rsm(bbr, nrsm, rsm, c_end);
5549         TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
5550         nrsm->r_dupack = 0;
5551         if (rsm->r_in_tmap) {
5552                 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
5553                 nrsm->r_in_tmap = 1;
5554         }
5555         rsm->r_flags &= (~BBR_HAS_FIN);
5556         bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
5557         *lenp = 0;
5558         return (0);
5559 }
5560
5561 static uint64_t
5562 bbr_get_hardware_rate(struct tcp_bbr *bbr)
5563 {
5564         uint64_t bw;
5565
5566         bw = bbr_get_bw(bbr);
5567         bw *= (uint64_t)bbr_hptsi_gain[BBR_SUB_GAIN];
5568         bw /= (uint64_t)BBR_UNIT;
5569         return(bw);
5570 }
5571
5572 static void
5573 bbr_setup_less_of_rate(struct tcp_bbr *bbr, uint32_t cts,
5574                        uint64_t act_rate, uint64_t rate_wanted)
5575 {
5576         /*
5577          * We could not get a full gains worth
5578          * of rate.
5579          */
5580         if (get_filter_value(&bbr->r_ctl.rc_delrate) >= act_rate) {
5581                 /* we can't even get the real rate */
5582                 uint64_t red;
5583
5584                 bbr->skip_gain = 1;
5585                 bbr->gain_is_limited = 0;
5586                 red = get_filter_value(&bbr->r_ctl.rc_delrate) - act_rate;
5587                 if (red)
5588                         filter_reduce_by(&bbr->r_ctl.rc_delrate, red, cts);
5589         } else {
5590                 /* We can use a lower gain */
5591                 bbr->skip_gain = 0;
5592                 bbr->gain_is_limited = 1;
5593         }
5594 }
5595
5596 static void
5597 bbr_update_hardware_pacing_rate(struct tcp_bbr *bbr, uint32_t cts)
5598 {
5599         const struct tcp_hwrate_limit_table *nrte;
5600         int error, rate = -1;
5601
5602         if (bbr->r_ctl.crte == NULL)
5603                 return;
5604         if ((bbr->rc_inp->inp_route.ro_rt == NULL) ||
5605             (bbr->rc_inp->inp_route.ro_rt->rt_ifp == NULL)) {
5606                 /* Lost our routes? */
5607                 /* Clear the way for a re-attempt */
5608                 bbr->bbr_attempt_hdwr_pace = 0;
5609 lost_rate:
5610                 bbr->gain_is_limited = 0;
5611                 bbr->skip_gain = 0;
5612                 bbr->bbr_hdrw_pacing = 0;
5613                 counter_u64_add(bbr_flows_whdwr_pacing, -1);
5614                 counter_u64_add(bbr_flows_nohdwr_pacing, 1);
5615                 tcp_bbr_tso_size_check(bbr, cts);
5616                 return;
5617         }
5618         rate = bbr_get_hardware_rate(bbr);
5619         nrte = tcp_chg_pacing_rate(bbr->r_ctl.crte,
5620                                    bbr->rc_tp,
5621                                    bbr->rc_inp->inp_route.ro_rt->rt_ifp,
5622                                    rate,
5623                                    (RS_PACING_GEQ|RS_PACING_SUB_OK),
5624                                    &error);
5625         if (nrte == NULL) {
5626                 goto lost_rate;
5627         }
5628         if (nrte != bbr->r_ctl.crte) {
5629                 bbr->r_ctl.crte = nrte;
5630                 if (error == 0)  {
5631                         BBR_STAT_INC(bbr_hdwr_rl_mod_ok);
5632                         if (bbr->r_ctl.crte->rate < rate) {
5633                                 /* We have a problem */
5634                                 bbr_setup_less_of_rate(bbr, cts,
5635                                                        bbr->r_ctl.crte->rate, rate);
5636                         } else {
5637                                 /* We are good */
5638                                 bbr->gain_is_limited = 0;
5639                                 bbr->skip_gain = 0;
5640                         }
5641                 } else {
5642                         /* A failure should release the tag */
5643                         BBR_STAT_INC(bbr_hdwr_rl_mod_fail);
5644                         bbr->gain_is_limited = 0;
5645                         bbr->skip_gain = 0;
5646                         bbr->bbr_hdrw_pacing = 0;
5647                 }
5648                 bbr_type_log_hdwr_pacing(bbr,
5649                                          bbr->r_ctl.crte->ptbl->rs_ifp,
5650                                          rate,
5651                                          ((bbr->r_ctl.crte == NULL) ? 0 : bbr->r_ctl.crte->rate),
5652                                          __LINE__,
5653                                          cts,
5654                                          error);
5655         }
5656 }
5657
5658 static void
5659 bbr_adjust_for_hw_pacing(struct tcp_bbr *bbr, uint32_t cts)
5660 {
5661         /*
5662          * If we have hardware pacing support
5663          * we need to factor that in for our
5664          * TSO size.
5665          */
5666         const struct tcp_hwrate_limit_table *rlp;
5667         uint32_t cur_delay, seg_sz, maxseg, new_tso, delta, hdwr_delay;
5668
5669         if ((bbr->bbr_hdrw_pacing == 0) ||
5670             (IN_RECOVERY(bbr->rc_tp->t_flags)) ||
5671             (bbr->r_ctl.crte == NULL))
5672                 return;
5673         if (bbr->hw_pacing_set == 0) {
5674                 /* Not yet by the hdwr pacing count delay */
5675                 return;
5676         }
5677         if (bbr_hdwr_pace_adjust == 0) {
5678                 /* No adjustment */
5679                 return;
5680         }
5681         rlp = bbr->r_ctl.crte;
5682         if (bbr->rc_tp->t_maxseg > bbr->rc_last_options)
5683                 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5684         else
5685                 maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5686         /*
5687          * So lets first get the
5688          * time we will take between
5689          * TSO sized sends currently without
5690          * hardware help.
5691          */
5692         cur_delay = bbr_get_pacing_delay(bbr, BBR_UNIT,
5693                         bbr->r_ctl.rc_pace_max_segs, cts, 1);
5694         hdwr_delay = bbr->r_ctl.rc_pace_max_segs / maxseg;
5695         hdwr_delay *= rlp->time_between;
5696         if (cur_delay > hdwr_delay)
5697                 delta = cur_delay - hdwr_delay;
5698         else
5699                 delta = 0;
5700         bbr_log_type_tsosize(bbr, cts, delta, cur_delay, hdwr_delay,
5701                              (bbr->r_ctl.rc_pace_max_segs / maxseg),
5702                              1);
5703         if (delta &&
5704             (delta < (max(rlp->time_between,
5705                           bbr->r_ctl.bbr_hptsi_segments_delay_tar)))) {
5706                 /*
5707                  * Now lets divide by the pacing
5708                  * time between each segment the
5709                  * hardware sends rounding up and
5710                  * derive a bytes from that. We multiply
5711                  * that by bbr_hdwr_pace_adjust to get
5712                  * more bang for our buck.
5713                  *
5714                  * The goal is to have the software pacer
5715                  * waiting no more than an additional
5716                  * pacing delay if we can (without the
5717                  * compensation i.e. x bbr_hdwr_pace_adjust).
5718                  */
5719                 seg_sz = max(((cur_delay + rlp->time_between)/rlp->time_between),
5720                              (bbr->r_ctl.rc_pace_max_segs/maxseg));
5721                 seg_sz *= bbr_hdwr_pace_adjust;
5722                 if (bbr_hdwr_pace_floor &&
5723                     (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5724                         /* Currently hardware paces
5725                          * out rs_min_seg segments at a time.
5726                          * We need to make sure we always send at least
5727                          * a full burst of bbr_hdwr_pace_floor down.
5728                          */
5729                         seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5730                 }
5731                 seg_sz *= maxseg;
5732         } else if (delta == 0) {
5733                 /*
5734                  * The highest pacing rate is
5735                  * above our b/w gained. This means
5736                  * we probably are going quite fast at
5737                  * the hardware highest rate. Lets just multiply
5738                  * the calculated TSO size by the
5739                  * multiplier factor (its probably
5740                  * 4 segments in the default config for
5741                  * mlx).
5742                  */
5743                 seg_sz = bbr->r_ctl.rc_pace_max_segs * bbr_hdwr_pace_adjust;
5744                 if (bbr_hdwr_pace_floor &&
5745                     (seg_sz < bbr->r_ctl.crte->ptbl->rs_min_seg)) {
5746                         /* Currently hardware paces
5747                          * out rs_min_seg segments at a time.
5748                          * We need to make sure we always send at least
5749                          * a full burst of bbr_hdwr_pace_floor down.
5750                          */
5751                         seg_sz = bbr->r_ctl.crte->ptbl->rs_min_seg;
5752                 }
5753         } else {
5754                 /*
5755                  * The pacing time difference is so
5756                  * big that the hardware will
5757                  * pace out more rapidly then we
5758                  * really want and then we
5759                  * will have a long delay. Lets just keep
5760                  * the same TSO size so its as if
5761                  * we were not using hdwr pacing (we
5762                  * just gain a bit of spacing from the
5763                  * hardware if seg_sz > 1).
5764                  */
5765                 seg_sz = bbr->r_ctl.rc_pace_max_segs;
5766         }
5767         if (seg_sz > bbr->r_ctl.rc_pace_max_segs)
5768                 new_tso = seg_sz;
5769         else
5770                 new_tso = bbr->r_ctl.rc_pace_max_segs;
5771         if (new_tso >= (PACE_MAX_IP_BYTES-maxseg))
5772                 new_tso = PACE_MAX_IP_BYTES - maxseg;
5773
5774         if (new_tso != bbr->r_ctl.rc_pace_max_segs) {
5775                 bbr_log_type_tsosize(bbr, cts, new_tso, 0, bbr->r_ctl.rc_pace_max_segs, maxseg, 0);
5776                 bbr->r_ctl.rc_pace_max_segs = new_tso;
5777         }
5778 }
5779
5780 static void
5781 tcp_bbr_tso_size_check(struct tcp_bbr *bbr, uint32_t cts)
5782 {
5783         uint64_t bw;
5784         uint32_t old_tso = 0, new_tso;
5785         uint32_t maxseg, bytes;
5786         uint32_t tls_seg=0;
5787         /*
5788          * Google/linux uses the following algorithm to determine
5789          * the TSO size based on the b/w of the link (from Neal Cardwell email 9/27/18):
5790          *
5791          *  bytes = bw_in_bytes_per_second / 1000
5792          *  bytes = min(bytes, 64k)
5793          *  tso_segs = bytes / MSS
5794          *  if (bw < 1.2Mbs)
5795          *      min_tso_segs = 1
5796          *  else
5797          *      min_tso_segs = 2
5798          * tso_segs = max(tso_segs, min_tso_segs)
5799          *
5800          * * Note apply a device specific limit (we apply this in the
5801          *   tcp_m_copym).
5802          * Note that before the initial measurement is made google bursts out
5803          * a full iwnd just like new-reno/cubic.
5804          *
5805          * We do not use this algorithm. Instead we
5806          * use a two phased approach:
5807          *
5808          *  if ( bw <= per-tcb-cross-over)
5809          *     goal_tso =  calculate how much with this bw we
5810          *                 can send in goal-time seconds.
5811          *     if (goal_tso > mss)
5812          *         seg = goal_tso / mss
5813          *         tso = seg * mss
5814          *     else
5815          *         tso = mss
5816          *     if (tso > per-tcb-max)
5817          *         tso = per-tcb-max
5818          *  else if ( bw > 512Mbps)
5819          *     tso = max-tso (64k/mss)
5820          *  else
5821          *     goal_tso = bw / per-tcb-divsor
5822          *     seg = (goal_tso + mss-1)/mss
5823          *     tso = seg * mss
5824          *
5825          * if (tso < per-tcb-floor)
5826          *    tso = per-tcb-floor
5827          * if (tso > per-tcb-utter_max)
5828          *    tso = per-tcb-utter_max
5829          *
5830          * Note the default per-tcb-divisor is 1000 (same as google).
5831          * the goal cross over is 30Mbps however. To recreate googles
5832          * algorithm you need to set:
5833          *
5834          * cross-over = 23,168,000 bps
5835          * goal-time = 18000
5836          * per-tcb-max = 2
5837          * per-tcb-divisor = 1000
5838          * per-tcb-floor = 1
5839          *
5840          * This will get you "google bbr" behavior with respect to tso size.
5841          *
5842          * Note we do set anything TSO size until we are past the initial
5843          * window. Before that we gnerally use either a single MSS
5844          * or we use the full IW size (so we burst a IW at a time)
5845          * Also note that Hardware-TLS is special and does alternate
5846          * things to minimize PCI Bus Bandwidth use.
5847          */
5848
5849         if (bbr->rc_tp->t_maxseg > bbr->rc_last_options) {
5850                 maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5851         } else {
5852                 maxseg = BBR_MIN_SEG - bbr->rc_last_options;
5853         }
5854 #ifdef KERN_TLS
5855         if (bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) {
5856                 tls_seg =  ctf_get_opt_tls_size(bbr->rc_inp->inp_socket, bbr->rc_tp->snd_wnd);
5857                 bbr->r_ctl.rc_pace_min_segs = (tls_seg + bbr->rc_last_options);
5858         }
5859 #endif
5860         old_tso = bbr->r_ctl.rc_pace_max_segs;
5861         if (bbr->rc_past_init_win == 0) {
5862                 /*
5863                  * Not enough data has been acknowledged to make a
5864                  * judgement unless we are hardware TLS. Set up
5865                  * the initial TSO based on if we are sending a
5866                  * full IW at once or not.
5867                  */
5868                 if (bbr->rc_use_google)
5869                         bbr->r_ctl.rc_pace_max_segs = ((bbr->rc_tp->t_maxseg - bbr->rc_last_options) * 2);
5870                 else if (bbr->bbr_init_win_cheat)
5871                         bbr->r_ctl.rc_pace_max_segs = bbr_initial_cwnd(bbr, bbr->rc_tp);
5872                 else
5873                         bbr->r_ctl.rc_pace_max_segs = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
5874                 if (bbr->r_ctl.rc_pace_min_segs != bbr->rc_tp->t_maxseg)
5875                         bbr->r_ctl.rc_pace_min_segs = bbr->rc_tp->t_maxseg;
5876 #ifdef KERN_TLS
5877                 if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) && tls_seg) {
5878                         /*
5879                          * For hardware TLS we set our min to the tls_seg size.
5880                          */
5881                         bbr->r_ctl.rc_pace_max_segs = tls_seg;
5882                         bbr->r_ctl.rc_pace_min_segs = tls_seg + bbr->rc_last_options;
5883                 }
5884 #endif
5885                 if (bbr->r_ctl.rc_pace_max_segs == 0) {
5886                         bbr->r_ctl.rc_pace_max_segs = maxseg;
5887                 }
5888                 bbr_log_type_tsosize(bbr, cts, bbr->r_ctl.rc_pace_max_segs, tls_seg, old_tso, maxseg, 0);
5889 #ifdef KERN_TLS
5890                 if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) == 0)
5891 #endif
5892                         bbr_adjust_for_hw_pacing(bbr, cts);
5893                 return;
5894         }
5895         /**
5896          * Now lets set the TSO goal based on our delivery rate in
5897          * bytes per second. Note we only do this if
5898          * we have acked at least the initial cwnd worth of data.
5899          */
5900         bw = bbr_get_bw(bbr);
5901         if (IN_RECOVERY(bbr->rc_tp->t_flags) &&
5902              (bbr->rc_use_google == 0)) {
5903                 /* We clamp to one MSS in recovery */
5904                 new_tso = maxseg;
5905         } else if (bbr->rc_use_google) {
5906                 int min_tso_segs;
5907
5908                 /* Google considers the gain too */
5909                 if (bbr->r_ctl.rc_bbr_hptsi_gain != BBR_UNIT) {
5910                         bw *= bbr->r_ctl.rc_bbr_hptsi_gain;
5911                         bw /= BBR_UNIT;
5912                 }
5913                 bytes = bw / 1024;
5914                 if (bytes > (64 * 1024))
5915                         bytes = 64 * 1024;
5916                 new_tso = bytes / maxseg;
5917                 if (bw < ONE_POINT_TWO_MEG)
5918                         min_tso_segs = 1;
5919                 else
5920                         min_tso_segs = 2;
5921                 if (new_tso < min_tso_segs)
5922                         new_tso = min_tso_segs;
5923                 new_tso *= maxseg;
5924         } else if (bbr->rc_no_pacing) {
5925                 new_tso = (PACE_MAX_IP_BYTES / maxseg) * maxseg;
5926         } else if (bw <= bbr->r_ctl.bbr_cross_over) {
5927                 /*
5928                  * Calculate the worse case b/w TSO if we are inserting no
5929                  * more than a delay_target number of TSO's.
5930                  */
5931                 uint32_t tso_len, min_tso;
5932
5933                 tso_len = bbr_get_pacing_length(bbr, BBR_UNIT, bbr->r_ctl.bbr_hptsi_segments_delay_tar, bw);
5934                 if (tso_len > maxseg) {
5935                         new_tso = tso_len / maxseg;
5936                         if (new_tso > bbr->r_ctl.bbr_hptsi_segments_max)
5937                                 new_tso = bbr->r_ctl.bbr_hptsi_segments_max;
5938                         new_tso *= maxseg;
5939                 } else {
5940                         /*
5941                          * less than a full sized frame yikes.. long rtt or
5942                          * low bw?
5943                          */
5944                         min_tso = bbr_minseg(bbr);
5945                         if ((tso_len > min_tso) && (bbr_all_get_min == 0))
5946                                 new_tso = rounddown(tso_len, min_tso);
5947                         else
5948                                 new_tso = min_tso;
5949                 }
5950         } else if (bw > FIVETWELVE_MBPS) {
5951                 /*
5952                  * This guy is so fast b/w wise that we can TSO as large as
5953                  * possible of segments that the NIC will allow.
5954                  */
5955                 new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg);
5956         } else {
5957                 /*
5958                  * This formula is based on attempting to send a segment or
5959                  * more every bbr_hptsi_per_second. The default is 1000
5960                  * which means you are targeting what you can send every 1ms
5961                  * based on the peers bw.
5962                  *
5963                  * If the number drops to say 500, then you are looking more
5964                  * at 2ms and you will raise how much we send in a single
5965                  * TSO thus saving CPU (less bbr_output_wtime() calls). The
5966                  * trade off of course is you will send more at once and
5967                  * thus tend to clump up the sends into larger "bursts"
5968                  * building a queue.
5969                  */
5970                 bw /= bbr->r_ctl.bbr_hptsi_per_second;
5971                 new_tso = roundup(bw, (uint64_t)maxseg);
5972                 /*
5973                  * Gate the floor to match what our lower than 48Mbps
5974                  * algorithm does. The ceiling (bbr_hptsi_segments_max) thus
5975                  * becomes the floor for this calculation.
5976                  */
5977                 if (new_tso < (bbr->r_ctl.bbr_hptsi_segments_max * maxseg))
5978                         new_tso = (bbr->r_ctl.bbr_hptsi_segments_max * maxseg);
5979         }
5980         if (bbr->r_ctl.bbr_hptsi_segments_floor && (new_tso < (maxseg * bbr->r_ctl.bbr_hptsi_segments_floor)))
5981                 new_tso = maxseg * bbr->r_ctl.bbr_hptsi_segments_floor;
5982         if (new_tso > PACE_MAX_IP_BYTES)
5983                 new_tso = rounddown(PACE_MAX_IP_BYTES, maxseg);
5984         /* Enforce an utter maximum if we are not HW-TLS */
5985 #ifdef KERN_TLS
5986         if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) == 0)
5987 #endif
5988                 if (bbr->r_ctl.bbr_utter_max && (new_tso > (bbr->r_ctl.bbr_utter_max * maxseg))) {
5989                         new_tso = bbr->r_ctl.bbr_utter_max * maxseg;
5990                 }
5991 #ifdef KERN_TLS
5992         if (tls_seg) {
5993                 /*
5994                  * Lets move the output size
5995                  * up to 1 or more TLS record sizes.
5996                  */
5997                 uint32_t temp;
5998
5999                 temp = roundup(new_tso, tls_seg);
6000                 new_tso = temp;
6001                 /* Back down if needed to under a full frame */
6002                 while (new_tso > PACE_MAX_IP_BYTES)
6003                         new_tso -= tls_seg;
6004         }
6005 #endif
6006         if (old_tso != new_tso) {
6007                 /* Only log changes */
6008                 bbr_log_type_tsosize(bbr, cts, new_tso, tls_seg, old_tso, maxseg, 0);
6009                 bbr->r_ctl.rc_pace_max_segs = new_tso;
6010         }
6011 #ifdef KERN_TLS
6012         if ((bbr->rc_inp->inp_socket->so_snd.sb_flags & SB_TLS_IFNET) &&
6013              tls_seg) {
6014                 bbr->r_ctl.rc_pace_min_segs = tls_seg + bbr->rc_last_options;
6015         } else
6016 #endif
6017                 /* We have hardware pacing and not hardware TLS! */
6018                 bbr_adjust_for_hw_pacing(bbr, cts);
6019 }
6020
6021 static void
6022 bbr_log_output(struct tcp_bbr *bbr, struct tcpcb *tp, struct tcpopt *to, int32_t len,
6023     uint32_t seq_out, uint8_t th_flags, int32_t err, uint32_t cts,
6024     struct mbuf *mb, int32_t * abandon, struct bbr_sendmap *hintrsm, uint32_t delay_calc,
6025     struct sockbuf *sb)
6026 {
6027
6028         struct bbr_sendmap *rsm, *nrsm;
6029         register uint32_t snd_max, snd_una;
6030         uint32_t pacing_time;
6031         /*
6032          * Add to the RACK log of packets in flight or retransmitted. If
6033          * there is a TS option we will use the TS echoed, if not we will
6034          * grab a TS.
6035          *
6036          * Retransmissions will increment the count and move the ts to its
6037          * proper place. Note that if options do not include TS's then we
6038          * won't be able to effectively use the ACK for an RTT on a retran.
6039          *
6040          * Notes about r_start and r_end. Lets consider a send starting at
6041          * sequence 1 for 10 bytes. In such an example the r_start would be
6042          * 1 (starting sequence) but the r_end would be r_start+len i.e. 11.
6043          * This means that r_end is actually the first sequence for the next
6044          * slot (11).
6045          *
6046          */
6047         INP_WLOCK_ASSERT(tp->t_inpcb);
6048         if (err) {
6049                 /*
6050                  * We don't log errors -- we could but snd_max does not
6051                  * advance in this case either.
6052                  */
6053                 return;
6054         }
6055         if (th_flags & TH_RST) {
6056                 /*
6057                  * We don't log resets and we return immediately from
6058                  * sending
6059                  */
6060                 *abandon = 1;
6061                 return;
6062         }
6063         snd_una = tp->snd_una;
6064         if (th_flags & (TH_SYN | TH_FIN) && (hintrsm == NULL)) {
6065                 /*
6066                  * The call to bbr_log_output is made before bumping
6067                  * snd_max. This means we can record one extra byte on a SYN
6068                  * or FIN if seq_out is adding more on and a FIN is present
6069                  * (and we are not resending).
6070                  */
6071                 if (th_flags & TH_SYN)
6072                         len++;
6073                 if (th_flags & TH_FIN)
6074                         len++;
6075         }
6076         if (SEQ_LEQ((seq_out + len), snd_una)) {
6077                 /* Are sending an old segment to induce an ack (keep-alive)? */
6078                 return;
6079         }
6080         if (SEQ_LT(seq_out, snd_una)) {
6081                 /* huh? should we panic? */
6082                 uint32_t end;
6083
6084                 end = seq_out + len;
6085                 seq_out = snd_una;
6086                 len = end - seq_out;
6087         }
6088         snd_max = tp->snd_max;
6089         if (len == 0) {
6090                 /* We don't log zero window probes */
6091                 return;
6092         }
6093         pacing_time = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, len, cts, 1);
6094         /* First question is it a retransmission? */
6095         if (seq_out == snd_max) {
6096 again:
6097                 rsm = bbr_alloc(bbr);
6098                 if (rsm == NULL) {
6099                         return;
6100                 }
6101                 rsm->r_flags = 0;
6102                 if (th_flags & TH_SYN)
6103                         rsm->r_flags |= BBR_HAS_SYN;
6104                 if (th_flags & TH_FIN)
6105                         rsm->r_flags |= BBR_HAS_FIN;
6106                 rsm->r_tim_lastsent[0] = cts;
6107                 rsm->r_rtr_cnt = 1;
6108                 rsm->r_rtr_bytes = 0;
6109                 rsm->r_start = seq_out;
6110                 rsm->r_end = rsm->r_start + len;
6111                 rsm->r_dupack = 0;
6112                 rsm->r_delivered = bbr->r_ctl.rc_delivered;
6113                 rsm->r_pacing_delay = pacing_time;
6114                 rsm->r_ts_valid = bbr->rc_ts_valid;
6115                 if (bbr->rc_ts_valid)
6116                         rsm->r_del_ack_ts = bbr->r_ctl.last_inbound_ts;
6117                 rsm->r_del_time = bbr->r_ctl.rc_del_time;
6118                 if (bbr->r_ctl.r_app_limited_until)
6119                         rsm->r_app_limited = 1;
6120                 else
6121                         rsm->r_app_limited = 0;
6122                 rsm->r_first_sent_time = bbr_get_earliest_send_outstanding(bbr, rsm, cts);
6123                 rsm->r_flight_at_send = ctf_flight_size(bbr->rc_tp,
6124                                                 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
6125                 /*
6126                  * Here we must also add in this rsm since snd_max
6127                  * is updated after we return from a new send.
6128                  */
6129                 rsm->r_flight_at_send += len;
6130                 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
6131                 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
6132                 rsm->r_in_tmap = 1;
6133                 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
6134                         rsm->r_bbr_state = bbr_state_val(bbr);
6135                 else
6136                         rsm->r_bbr_state = 8;
6137                 if (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT) {
6138                         rsm->r_is_gain = 1;
6139                         rsm->r_is_drain = 0;
6140                 } else if (bbr->r_ctl.rc_bbr_hptsi_gain < BBR_UNIT) {
6141                         rsm->r_is_drain = 1;
6142                         rsm->r_is_gain = 0;
6143                 } else {
6144                         rsm->r_is_drain = 0;
6145                         rsm->r_is_gain = 0;
6146                 }
6147                 return;
6148         }
6149         /*
6150          * If we reach here its a retransmission and we need to find it.
6151          */
6152 more:
6153         if (hintrsm && (hintrsm->r_start == seq_out)) {
6154                 rsm = hintrsm;
6155                 hintrsm = NULL;
6156         } else if (bbr->r_ctl.rc_next) {
6157                 /* We have a hint from a previous run */
6158                 rsm = bbr->r_ctl.rc_next;
6159         } else {
6160                 /* No hints sorry */
6161                 rsm = NULL;
6162         }
6163         if ((rsm) && (rsm->r_start == seq_out)) {
6164                 /*
6165                  * We used rc_next or hintrsm  to retransmit, hopefully the
6166                  * likely case.
6167                  */
6168                 seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6169                 if (len == 0) {
6170                         return;
6171                 } else {
6172                         goto more;
6173                 }
6174         }
6175         /* Ok it was not the last pointer go through it the hard way. */
6176         TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6177                 if (rsm->r_start == seq_out) {
6178                         seq_out = bbr_update_entry(tp, bbr, rsm, cts, &len, pacing_time);
6179                         bbr->r_ctl.rc_next = TAILQ_NEXT(rsm, r_next);
6180                         if (len == 0) {
6181                                 return;
6182                         } else {
6183                                 continue;
6184                         }
6185                 }
6186                 if (SEQ_GEQ(seq_out, rsm->r_start) && SEQ_LT(seq_out, rsm->r_end)) {
6187                         /* Transmitted within this piece */
6188                         /*
6189                          * Ok we must split off the front and then let the
6190                          * update do the rest
6191                          */
6192                         nrsm = bbr_alloc_full_limit(bbr);
6193                         if (nrsm == NULL) {
6194                                 bbr_update_rsm(tp, bbr, rsm, cts, pacing_time);
6195                                 return;
6196                         }
6197                         /*
6198                          * copy rsm to nrsm and then trim the front of rsm
6199                          * to not include this part.
6200                          */
6201                         bbr_clone_rsm(bbr, nrsm, rsm, seq_out);
6202                         TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
6203                         if (rsm->r_in_tmap) {
6204                                 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
6205                                 nrsm->r_in_tmap = 1;
6206                         }
6207                         rsm->r_flags &= (~BBR_HAS_FIN);
6208                         seq_out = bbr_update_entry(tp, bbr, nrsm, cts, &len, pacing_time);
6209                         if (len == 0) {
6210                                 return;
6211                         }
6212                 }
6213         }
6214         /*
6215          * Hmm not found in map did they retransmit both old and on into the
6216          * new?
6217          */
6218         if (seq_out == tp->snd_max) {
6219                 goto again;
6220         } else if (SEQ_LT(seq_out, tp->snd_max)) {
6221 #ifdef BBR_INVARIANTS
6222                 printf("seq_out:%u len:%d snd_una:%u snd_max:%u -- but rsm not found?\n",
6223                     seq_out, len, tp->snd_una, tp->snd_max);
6224                 printf("Starting Dump of all rack entries\n");
6225                 TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
6226                         printf("rsm:%p start:%u end:%u\n",
6227                             rsm, rsm->r_start, rsm->r_end);
6228                 }
6229                 printf("Dump complete\n");
6230                 panic("seq_out not found rack:%p tp:%p",
6231                     bbr, tp);
6232 #endif
6233         } else {
6234 #ifdef BBR_INVARIANTS
6235                 /*
6236                  * Hmm beyond sndmax? (only if we are using the new rtt-pack
6237                  * flag)
6238                  */
6239                 panic("seq_out:%u(%d) is beyond snd_max:%u tp:%p",
6240                     seq_out, len, tp->snd_max, tp);
6241 #endif
6242         }
6243 }
6244
6245 static void
6246 bbr_collapse_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, int32_t rtt)
6247 {
6248         /*
6249          * Collapse timeout back the cum-ack moved.
6250          */
6251         tp->t_rxtshift = 0;
6252         tp->t_softerror = 0;
6253 }
6254
6255
6256 static void
6257 tcp_bbr_xmit_timer(struct tcp_bbr *bbr, uint32_t rtt_usecs, uint32_t rsm_send_time, uint32_t r_start, uint32_t tsin)
6258 {
6259         bbr->rtt_valid = 1;
6260         bbr->r_ctl.cur_rtt = rtt_usecs;
6261         bbr->r_ctl.ts_in = tsin;
6262         if (rsm_send_time)
6263                 bbr->r_ctl.cur_rtt_send_time = rsm_send_time;
6264 }
6265
6266 static void
6267 bbr_make_timestamp_determination(struct tcp_bbr *bbr)
6268 {
6269         /**
6270          * We have in our bbr control:
6271          * 1) The timestamp we started observing cum-acks (bbr->r_ctl.bbr_ts_check_tstmp).
6272          * 2) Our timestamp indicating when we sent that packet (bbr->r_ctl.rsm->bbr_ts_check_our_cts).
6273          * 3) The current timestamp that just came in (bbr->r_ctl.last_inbound_ts)
6274          * 4) The time that the packet that generated that ack was sent (bbr->r_ctl.cur_rtt_send_time)
6275          *
6276          * Now we can calculate the time between the sends by doing:
6277          *
6278          * delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts
6279          *
6280          * And the peer's time between receiving them by doing:
6281          *
6282          * peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp
6283          *
6284          * We want to figure out if the timestamp values are in msec, 10msec or usec.
6285          * We also may find that we can't use the timestamps if say we see
6286          * that the peer_delta indicates that though we may have taken 10ms to
6287          * pace out the data, it only saw 1ms between the two packets. This would
6288          * indicate that somewhere on the path is a batching entity that is giving
6289          * out time-slices of the actual b/w. This would mean we could not use
6290          * reliably the peers timestamps.
6291          *
6292          * We expect delta > peer_delta initially. Until we figure out the
6293          * timestamp difference which we will store in bbr->r_ctl.bbr_peer_tsratio.
6294          * If we place 1000 there then its a ms vs our usec. If we place 10000 there
6295          * then its 10ms vs our usec. If the peer is running a usec clock we would
6296          * put a 1 there. If the value is faster then ours, we will disable the
6297          * use of timestamps (though we could revist this later if we find it to be not
6298          * just an isolated one or two flows)).
6299          *
6300          * To detect the batching middle boxes we will come up with our compensation and
6301          * if with it in place, we find the peer is drastically off (by some margin) in
6302          * the smaller direction, then we will assume the worst case and disable use of timestamps.
6303          *
6304          */
6305         uint64_t delta, peer_delta, delta_up;
6306
6307         delta = bbr->r_ctl.cur_rtt_send_time - bbr->r_ctl.bbr_ts_check_our_cts;
6308         if (delta < bbr_min_usec_delta) {
6309                 /*
6310                  * Have not seen a min amount of time
6311                  * between our send times so we can
6312                  * make a determination of the timestamp
6313                  * yet.
6314                  */
6315                 return;
6316         }
6317         peer_delta = bbr->r_ctl.last_inbound_ts - bbr->r_ctl.bbr_ts_check_tstmp;
6318         if (peer_delta < bbr_min_peer_delta) {
6319                 /*
6320                  * We may have enough in the form of
6321                  * our delta but the peers number
6322                  * has not changed that much. It could
6323                  * be its clock ratio is such that
6324                  * we need more data (10ms tick) or
6325                  * there may be other compression scenarios
6326                  * going on. In any event we need the
6327                  * spread to be larger.
6328                  */
6329                 return;
6330         }
6331         /* Ok lets first see which way our delta is going */
6332         if (peer_delta > delta) {
6333                 /* Very unlikely, the peer without
6334                  * compensation shows that it saw
6335                  * the two sends arrive further apart
6336                  * then we saw then in micro-seconds.
6337                  */
6338                 if (peer_delta < (delta + ((delta * (uint64_t)1000)/ (uint64_t)bbr_delta_percent))) {
6339                         /* well it looks like the peer is a micro-second clock. */
6340                         bbr->rc_ts_clock_set = 1;
6341                         bbr->r_ctl.bbr_peer_tsratio = 1;
6342                 } else {
6343                         bbr->rc_ts_cant_be_used = 1;
6344                         bbr->rc_ts_clock_set = 1;
6345                 }
6346                 return;
6347         }
6348         /* Ok we know that the peer_delta is smaller than our send distance */
6349         bbr->rc_ts_clock_set = 1;
6350         /* First question is it within the percentage that they are using usec time? */
6351         delta_up = (peer_delta * 1000) / (uint64_t)bbr_delta_percent;
6352         if ((peer_delta + delta_up) >= delta) {
6353                 /* Its a usec clock */
6354                 bbr->r_ctl.bbr_peer_tsratio = 1;
6355                 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6356                 return;
6357         }
6358         /* Ok if not usec, what about 10usec (though unlikely)? */
6359         delta_up = (peer_delta * 1000 * 10) / (uint64_t)bbr_delta_percent;
6360         if (((peer_delta * 10) + delta_up) >= delta) {
6361                 bbr->r_ctl.bbr_peer_tsratio = 10;
6362                 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6363                 return;
6364         }
6365         /* And what about 100usec (though again unlikely)? */
6366         delta_up = (peer_delta * 1000 * 100) / (uint64_t)bbr_delta_percent;
6367         if (((peer_delta * 100) + delta_up) >= delta) {
6368                 bbr->r_ctl.bbr_peer_tsratio = 100;
6369                 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6370                 return;
6371         }
6372         /* And how about 1 msec (the most likely one)? */
6373         delta_up = (peer_delta * 1000 * 1000) / (uint64_t)bbr_delta_percent;
6374         if (((peer_delta * 1000) + delta_up) >= delta) {
6375                 bbr->r_ctl.bbr_peer_tsratio = 1000;
6376                 bbr_log_tstmp_validation(bbr, peer_delta, delta);
6377                 return;
6378         }
6379         /* Ok if not msec could it be 10 msec? */
6380         delta_up = (peer_delta * 1000 * 10000) / (uint64_t)bbr_delta_percent;
6381         if (((peer_delta * 10000) + delta_up) >= delta) {
6382                 bbr->r_ctl.bbr_peer_tsratio = 10000;
6383                 return;
6384         }
6385         /* If we fall down here the clock tick so slowly we can't use it */
6386         bbr->rc_ts_cant_be_used = 1;
6387         bbr->r_ctl.bbr_peer_tsratio = 0;
6388         bbr_log_tstmp_validation(bbr, peer_delta, delta);
6389 }
6390
6391 /*
6392  * Collect new round-trip time estimate
6393  * and update averages and current timeout.
6394  */
6395 static void
6396 tcp_bbr_xmit_timer_commit(struct tcp_bbr *bbr, struct tcpcb *tp, uint32_t cts)
6397 {
6398         int32_t delta;
6399         uint32_t rtt, tsin;
6400         int32_t rtt_ticks;
6401
6402
6403         if (bbr->rtt_valid == 0)
6404                 /* No valid sample */
6405                 return;
6406
6407         rtt = bbr->r_ctl.cur_rtt;
6408         tsin = bbr->r_ctl.ts_in;
6409         if (bbr->rc_prtt_set_ts) {
6410                 /*
6411                  * We are to force feed the rttProp filter due
6412                  * to an entry into PROBE_RTT. This assures
6413                  * that the times are sync'd between when we
6414                  * go into PROBE_RTT and the filter expiration.
6415                  *
6416                  * Google does not use a true filter, so they do
6417                  * this implicitly since they only keep one value
6418                  * and when they enter probe-rtt they update the
6419                  * value to the newest rtt.
6420                  */
6421                 uint32_t rtt_prop;
6422
6423                 bbr->rc_prtt_set_ts = 0;
6424                 rtt_prop = get_filter_value_small(&bbr->r_ctl.rc_rttprop);
6425                 if (rtt > rtt_prop)
6426                         filter_increase_by_small(&bbr->r_ctl.rc_rttprop, (rtt - rtt_prop), cts);
6427                 else
6428                         apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6429         }
6430         if (bbr->rc_ack_was_delayed)
6431                 rtt += bbr->r_ctl.rc_ack_hdwr_delay;
6432
6433         if (rtt < bbr->r_ctl.rc_lowest_rtt)
6434                 bbr->r_ctl.rc_lowest_rtt = rtt;
6435         bbr_log_rtt_sample(bbr, rtt, tsin);
6436         if (bbr->r_init_rtt) {
6437                 /*
6438                  * The initial rtt is not-trusted, nuke it and lets get
6439                  * our first valid measurement in.
6440                  */
6441                 bbr->r_init_rtt = 0;
6442                 tp->t_srtt = 0;
6443         }
6444         if ((bbr->rc_ts_clock_set == 0) && bbr->rc_ts_valid) {
6445                 /*
6446                  * So we have not yet figured out
6447                  * what the peers TSTMP value is
6448                  * in (most likely ms). We need a
6449                  * series of cum-ack's to determine
6450                  * this reliably.
6451                  */
6452                 if (bbr->rc_ack_is_cumack) {
6453                         if (bbr->rc_ts_data_set) {
6454                                 /* Lets attempt to determine the timestamp granularity. */
6455                                 bbr_make_timestamp_determination(bbr);
6456                         } else {
6457                                 bbr->rc_ts_data_set = 1;
6458                                 bbr->r_ctl.bbr_ts_check_tstmp = bbr->r_ctl.last_inbound_ts;
6459                                 bbr->r_ctl.bbr_ts_check_our_cts = bbr->r_ctl.cur_rtt_send_time;
6460                         }
6461                 } else {
6462                         /*
6463                          * We have to have consecutive acks
6464                          * reset any "filled" state to none.
6465                          */
6466                         bbr->rc_ts_data_set = 0;
6467                 }
6468         }
6469         /* Round it up */
6470         rtt_ticks = USEC_2_TICKS((rtt + (USECS_IN_MSEC - 1)));
6471         if (rtt_ticks == 0)
6472                 rtt_ticks = 1;
6473         if (tp->t_srtt != 0) {
6474                 /*
6475                  * srtt is stored as fixed point with 5 bits after the
6476                  * binary point (i.e., scaled by 8).  The following magic is
6477                  * equivalent to the smoothing algorithm in rfc793 with an
6478                  * alpha of .875 (srtt = rtt/8 + srtt*7/8 in fixed point).
6479                  * Adjust rtt to origin 0.
6480                  */
6481
6482                 delta = ((rtt_ticks - 1) << TCP_DELTA_SHIFT)
6483                     - (tp->t_srtt >> (TCP_RTT_SHIFT - TCP_DELTA_SHIFT));
6484
6485                 tp->t_srtt += delta;
6486                 if (tp->t_srtt <= 0)
6487                         tp->t_srtt = 1;
6488
6489                 /*
6490                  * We accumulate a smoothed rtt variance (actually, a
6491                  * smoothed mean difference), then set the retransmit timer
6492                  * to smoothed rtt + 4 times the smoothed variance. rttvar
6493                  * is stored as fixed point with 4 bits after the binary
6494                  * point (scaled by 16).  The following is equivalent to
6495                  * rfc793 smoothing with an alpha of .75 (rttvar =
6496                  * rttvar*3/4 + |delta| / 4).  This replaces rfc793's
6497                  * wired-in beta.
6498                  */
6499                 if (delta < 0)
6500                         delta = -delta;
6501                 delta -= tp->t_rttvar >> (TCP_RTTVAR_SHIFT - TCP_DELTA_SHIFT);
6502                 tp->t_rttvar += delta;
6503                 if (tp->t_rttvar <= 0)
6504                         tp->t_rttvar = 1;
6505                 if (tp->t_rttbest > tp->t_srtt + tp->t_rttvar)
6506                         tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
6507         } else {
6508                 /*
6509                  * No rtt measurement yet - use the unsmoothed rtt. Set the
6510                  * variance to half the rtt (so our first retransmit happens
6511                  * at 3*rtt).
6512                  */
6513                 tp->t_srtt = rtt_ticks << TCP_RTT_SHIFT;
6514                 tp->t_rttvar = rtt_ticks << (TCP_RTTVAR_SHIFT - 1);
6515                 tp->t_rttbest = tp->t_srtt + tp->t_rttvar;
6516         }
6517         KMOD_TCPSTAT_INC(tcps_rttupdated);
6518         tp->t_rttupdated++;
6519 #ifdef STATS
6520         stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RTT, imax(0, rtt_ticks));
6521 #endif
6522         /*
6523          * the retransmit should happen at rtt + 4 * rttvar. Because of the
6524          * way we do the smoothing, srtt and rttvar will each average +1/2
6525          * tick of bias.  When we compute the retransmit timer, we want 1/2
6526          * tick of rounding and 1 extra tick because of +-1/2 tick
6527          * uncertainty in the firing of the timer.  The bias will give us
6528          * exactly the 1.5 tick we need.  But, because the bias is
6529          * statistical, we have to test that we don't drop below the minimum
6530          * feasible timer (which is 2 ticks).
6531          */
6532         TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
6533             max(MSEC_2_TICKS(bbr->r_ctl.rc_min_rto_ms), rtt_ticks + 2),
6534             MSEC_2_TICKS(((uint32_t)bbr->rc_max_rto_sec) * 1000));
6535
6536         /*
6537          * We received an ack for a packet that wasn't retransmitted; it is
6538          * probably safe to discard any error indications we've received
6539          * recently.  This isn't quite right, but close enough for now (a
6540          * route might have failed after we sent a segment, and the return
6541          * path might not be symmetrical).
6542          */
6543         tp->t_softerror = 0;
6544         rtt = (TICKS_2_USEC(bbr->rc_tp->t_srtt) >> TCP_RTT_SHIFT);
6545         if (bbr->r_ctl.bbr_smallest_srtt_this_state > rtt)
6546                 bbr->r_ctl.bbr_smallest_srtt_this_state = rtt;
6547 }
6548
6549 static void
6550 bbr_earlier_retran(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm,
6551                    uint32_t t, uint32_t cts, int ack_type)
6552 {
6553         /*
6554          * For this RSM, we acknowledged the data from a previous
6555          * transmission, not the last one we made. This means we did a false
6556          * retransmit.
6557          */
6558         if (rsm->r_flags & BBR_HAS_FIN) {
6559                 /*
6560                  * The sending of the FIN often is multiple sent when we
6561                  * have everything outstanding ack'd. We ignore this case
6562                  * since its over now.
6563                  */
6564                 return;
6565         }
6566         if (rsm->r_flags & BBR_TLP) {
6567                 /*
6568                  * We expect TLP's to have this occur often
6569                  */
6570                 bbr->rc_tlp_rtx_out = 0;
6571                 return;
6572         }
6573         if (ack_type != BBR_CUM_ACKED) {
6574                 /*
6575                  * If it was not a cum-ack we
6576                  * don't really know for sure since
6577                  * the timestamp could be from some
6578                  * other transmission.
6579                  */
6580                 return;
6581         }
6582
6583         if (rsm->r_flags & BBR_WAS_SACKPASS) {
6584                 /*
6585                  * We retransmitted based on a sack and the earlier
6586                  * retransmission ack'd it - re-ordering is occuring.
6587                  */
6588                 BBR_STAT_INC(bbr_reorder_seen);
6589                 bbr->r_ctl.rc_reorder_ts = cts;
6590         }
6591         /* Back down the loss count */
6592         if (rsm->r_flags & BBR_MARKED_LOST) {
6593                 bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
6594                 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
6595                 rsm->r_flags &= ~BBR_MARKED_LOST;
6596                 if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
6597                         /* LT sampling also needs adjustment */
6598                         bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
6599         }
6600         /***** RRS HERE ************************/
6601         /* Do we need to do this???            */
6602         /* bbr_reset_lt_bw_sampling(bbr, cts); */
6603         /***** RRS HERE ************************/
6604         BBR_STAT_INC(bbr_badfr);
6605         BBR_STAT_ADD(bbr_badfr_bytes, (rsm->r_end - rsm->r_start));
6606 }
6607
6608
6609 static void
6610 bbr_set_reduced_rtt(struct tcp_bbr *bbr, uint32_t cts, uint32_t line)
6611 {
6612         bbr->r_ctl.rc_rtt_shrinks = cts;
6613         if (bbr_can_force_probertt &&
6614             (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
6615             ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
6616                 /*
6617                  * We should enter probe-rtt its been too long
6618                  * since we have been there.
6619                  */
6620                 bbr_enter_probe_rtt(bbr, cts, __LINE__);
6621         } else
6622                 bbr_check_probe_rtt_limits(bbr, cts);
6623 }
6624
6625 static void
6626 tcp_bbr_commit_bw(struct tcp_bbr *bbr, uint32_t cts)
6627 {
6628         uint64_t orig_bw;
6629
6630         if (bbr->r_ctl.rc_bbr_cur_del_rate == 0) {
6631                 /* We never apply a zero measurment */
6632                 bbr_log_type_bbrupd(bbr, 20, cts, 0, 0,
6633                                     0, 0, 0, 0, 0, 0);
6634                 return;
6635         }
6636         if (bbr->r_ctl.r_measurement_count < 0xffffffff)
6637                 bbr->r_ctl.r_measurement_count++;
6638         orig_bw = get_filter_value(&bbr->r_ctl.rc_delrate);
6639         apply_filter_max(&bbr->r_ctl.rc_delrate, bbr->r_ctl.rc_bbr_cur_del_rate, bbr->r_ctl.rc_pkt_epoch);
6640         bbr_log_type_bbrupd(bbr, 21, cts, (uint32_t)orig_bw,
6641                             (uint32_t)get_filter_value(&bbr->r_ctl.rc_delrate),
6642                             0, 0, 0, 0, 0, 0);
6643         if (orig_bw &&
6644             (orig_bw != get_filter_value(&bbr->r_ctl.rc_delrate))) {
6645                 if (bbr->bbr_hdrw_pacing) {
6646                         /*
6647                          * Apply a new rate to the hardware
6648                          * possibly.
6649                          */
6650                         bbr_update_hardware_pacing_rate(bbr, cts);
6651                 }
6652                 bbr_set_state_target(bbr, __LINE__);
6653                 tcp_bbr_tso_size_check(bbr, cts);
6654                 if (bbr->r_recovery_bw)  {
6655                         bbr_setup_red_bw(bbr, cts);
6656                         bbr_log_type_bw_reduce(bbr, BBR_RED_BW_USELRBW);
6657                 }
6658         } else if ((orig_bw == 0) && get_filter_value(&bbr->r_ctl.rc_delrate))
6659                 tcp_bbr_tso_size_check(bbr, cts);
6660 }
6661
6662 static void
6663 bbr_nf_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6664 {
6665         if (bbr->rc_in_persist == 0) {
6666                 /* We log only when not in persist */
6667                 /* Translate to a Bytes Per Second */
6668                 uint64_t tim, bw, ts_diff, ts_bw;
6669                 uint32_t upper, lower, delivered;
6670
6671                 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6672                         tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6673                 else
6674                         tim = 1;
6675                 /*
6676                  * Now that we have processed the tim (skipping the sample
6677                  * or possibly updating the time, go ahead and
6678                  * calculate the cdr.
6679                  */
6680                 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6681                 bw = (uint64_t)delivered;
6682                 bw *= (uint64_t)USECS_IN_SECOND;
6683                 bw /= tim;
6684                 if (bw == 0) {
6685                         /* We must have a calculatable amount */
6686                         return;
6687                 }
6688                 upper = (bw >> 32) & 0x00000000ffffffff;
6689                 lower = bw & 0x00000000ffffffff;
6690                 /*
6691                  * If we are using this b/w shove it in now so we
6692                  * can see in the trace viewer if it gets over-ridden.
6693                  */
6694                 if (rsm->r_ts_valid &&
6695                     bbr->rc_ts_valid &&
6696                     bbr->rc_ts_clock_set &&
6697                     (bbr->rc_ts_cant_be_used == 0) &&
6698                     bbr->rc_use_ts_limit) {
6699                         ts_diff = max((bbr->r_ctl.last_inbound_ts - rsm->r_del_ack_ts), 1);
6700                         ts_diff *= bbr->r_ctl.bbr_peer_tsratio;
6701                         if ((delivered == 0) ||
6702                             (rtt < 1000)) {
6703                                 /* Can't use the ts */
6704                                 bbr_log_type_bbrupd(bbr, 61, cts,
6705                                                     ts_diff,
6706                                                     bbr->r_ctl.last_inbound_ts,
6707                                                     rsm->r_del_ack_ts, 0,
6708                                                     0, 0, 0, delivered);
6709                         } else {
6710                                 ts_bw = (uint64_t)delivered;
6711                                 ts_bw *= (uint64_t)USECS_IN_SECOND;
6712                                 ts_bw /= ts_diff;
6713                                 bbr_log_type_bbrupd(bbr, 62, cts,
6714                                                     (ts_bw >> 32),
6715                                                     (ts_bw & 0xffffffff), 0, 0,
6716                                                     0, 0, ts_diff, delivered);
6717                                 if ((bbr->ts_can_raise) &&
6718                                     (ts_bw > bw)) {
6719                                         bbr_log_type_bbrupd(bbr, 8, cts,
6720                                                             delivered,
6721                                                             ts_diff,
6722                                                             (bw >> 32),
6723                                                             (bw & 0x00000000ffffffff),
6724                                                             0, 0, 0, 0);
6725                                         bw = ts_bw;
6726                                 } else if (ts_bw && (ts_bw < bw)) {
6727                                         bbr_log_type_bbrupd(bbr, 7, cts,
6728                                                             delivered,
6729                                                             ts_diff,
6730                                                             (bw >> 32),
6731                                                             (bw & 0x00000000ffffffff),
6732                                                             0, 0, 0, 0);
6733                                         bw = ts_bw;
6734                                 }
6735                         }
6736                 }
6737                 if (rsm->r_first_sent_time &&
6738                     TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6739                         uint64_t sbw, sti;
6740                         /*
6741                          * We use what was in flight at the time of our
6742                          * send  and the size of this send to figure
6743                          * out what we have been sending at (amount).
6744                          * For the time we take from the time of
6745                          * the send of the first send outstanding
6746                          * until this send plus this sends pacing
6747                          * time. This gives us a good calculation
6748                          * as to the rate we have been sending at.
6749                          */
6750
6751                         sbw = (uint64_t)(rsm->r_flight_at_send);
6752                         sbw *= (uint64_t)USECS_IN_SECOND;
6753                         sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6754                         sti += rsm->r_pacing_delay;
6755                         sbw /= sti;
6756                         if (sbw < bw) {
6757                                 bbr_log_type_bbrupd(bbr, 6, cts,
6758                                                     delivered,
6759                                                     (uint32_t)sti,
6760                                                     (bw >> 32),
6761                                                     (uint32_t)bw,
6762                                                     rsm->r_first_sent_time, 0, (sbw >> 32),
6763                                                     (uint32_t)sbw);
6764                                 bw = sbw;
6765                         }
6766                 }
6767                 /* Use the google algorithm for b/w measurements */
6768                 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6769                 if ((rsm->r_app_limited == 0) ||
6770                     (bw > get_filter_value(&bbr->r_ctl.rc_delrate))) {
6771                         tcp_bbr_commit_bw(bbr, cts);
6772                         bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered,
6773                                             0, 0, 0, 0,  bbr->r_ctl.rc_del_time,  rsm->r_del_time);
6774                 }
6775         }
6776 }
6777
6778 static void
6779 bbr_google_measurement(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts)
6780 {
6781         if (bbr->rc_in_persist == 0) {
6782                 /* We log only when not in persist */
6783                 /* Translate to a Bytes Per Second */
6784                 uint64_t tim, bw;
6785                 uint32_t upper, lower, delivered;
6786                 int no_apply = 0;
6787
6788                 if (TSTMP_GT(bbr->r_ctl.rc_del_time, rsm->r_del_time))
6789                         tim = (uint64_t)(bbr->r_ctl.rc_del_time - rsm->r_del_time);
6790                 else
6791                         tim = 1;
6792                 /*
6793                  * Now that we have processed the tim (skipping the sample
6794                  * or possibly updating the time, go ahead and
6795                  * calculate the cdr.
6796                  */
6797                 delivered = (bbr->r_ctl.rc_delivered - rsm->r_delivered);
6798                 bw = (uint64_t)delivered;
6799                 bw *= (uint64_t)USECS_IN_SECOND;
6800                 bw /= tim;
6801                 if (tim < bbr->r_ctl.rc_lowest_rtt) {
6802                         bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered,
6803                                             tim, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6804
6805                         no_apply = 1;
6806                 }
6807                 upper = (bw >> 32) & 0x00000000ffffffff;
6808                 lower = bw & 0x00000000ffffffff;
6809                 /*
6810                  * If we are using this b/w shove it in now so we
6811                  * can see in the trace viewer if it gets over-ridden.
6812                  */
6813                 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6814                 /* Gate by the sending rate */
6815                 if (rsm->r_first_sent_time &&
6816                     TSTMP_GT(rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)],rsm->r_first_sent_time)) {
6817                         uint64_t sbw, sti;
6818                         /*
6819                          * We use what was in flight at the time of our
6820                          * send  and the size of this send to figure
6821                          * out what we have been sending at (amount).
6822                          * For the time we take from the time of
6823                          * the send of the first send outstanding
6824                          * until this send plus this sends pacing
6825                          * time. This gives us a good calculation
6826                          * as to the rate we have been sending at.
6827                          */
6828
6829                         sbw = (uint64_t)(rsm->r_flight_at_send);
6830                         sbw *= (uint64_t)USECS_IN_SECOND;
6831                         sti = rsm->r_tim_lastsent[(rsm->r_rtr_cnt -1)] - rsm->r_first_sent_time;
6832                         sti += rsm->r_pacing_delay;
6833                         sbw /= sti;
6834                         if (sbw < bw) {
6835                                 bbr_log_type_bbrupd(bbr, 6, cts,
6836                                                     delivered,
6837                                                     (uint32_t)sti,
6838                                                     (bw >> 32),
6839                                                     (uint32_t)bw,
6840                                                     rsm->r_first_sent_time, 0, (sbw >> 32),
6841                                                     (uint32_t)sbw);
6842                                 bw = sbw;
6843                         }
6844                         if ((sti > tim) &&
6845                             (sti < bbr->r_ctl.rc_lowest_rtt)) {
6846                                 bbr_log_type_bbrupd(bbr, 99, cts, (uint32_t)tim, delivered,
6847                                                     (uint32_t)sti, bbr->r_ctl.rc_lowest_rtt, 0, 0, 0, 0);
6848                                 no_apply = 1;
6849                         } else
6850                                 no_apply = 0;
6851                 }
6852                 bbr->r_ctl.rc_bbr_cur_del_rate = bw;
6853                 if ((no_apply == 0) &&
6854                     ((rsm->r_app_limited == 0) ||
6855                      (bw > get_filter_value(&bbr->r_ctl.rc_delrate)))) {
6856                         tcp_bbr_commit_bw(bbr, cts);
6857                         bbr_log_type_bbrupd(bbr, 10, cts, (uint32_t)tim, delivered,
6858                                             0, 0, 0, 0, bbr->r_ctl.rc_del_time,  rsm->r_del_time);
6859                 }
6860         }
6861 }
6862
6863
6864 static void
6865 bbr_update_bbr_info(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, uint32_t rtt, uint32_t cts, uint32_t tsin,
6866     uint32_t uts, int32_t match, uint32_t rsm_send_time, int32_t ack_type, struct tcpopt *to)
6867 {
6868         uint64_t old_rttprop;
6869
6870         /* Update our delivery time and amount */
6871         bbr->r_ctl.rc_delivered += (rsm->r_end - rsm->r_start);
6872         bbr->r_ctl.rc_del_time = cts;
6873         if (rtt == 0) {
6874                 /*
6875                  * 0 means its a retransmit, for now we don't use these for
6876                  * the rest of BBR.
6877                  */
6878                 return;
6879         }
6880         if ((bbr->rc_use_google == 0) &&
6881             (match != BBR_RTT_BY_EXACTMATCH) &&
6882             (match != BBR_RTT_BY_TIMESTAMP)){
6883                 /*
6884                  * We get a lot of rtt updates, lets not pay attention to
6885                  * any that are not an exact match. That way we don't have
6886                  * to worry about timestamps and the whole nonsense of
6887                  * unsure if its a retransmission etc (if we ever had the
6888                  * timestamp fixed to always have the last thing sent this
6889                  * would not be a issue).
6890                  */
6891                 return;
6892         }
6893         if ((bbr_no_retran && bbr->rc_use_google) &&
6894             (match != BBR_RTT_BY_EXACTMATCH) &&
6895             (match != BBR_RTT_BY_TIMESTAMP)){
6896                 /*
6897                  * We only do measurements in google mode
6898                  * with bbr_no_retran on for sure things.
6899                  */
6900                 return;
6901         }
6902         /* Only update srtt if we know by exact match */
6903         tcp_bbr_xmit_timer(bbr, rtt, rsm_send_time, rsm->r_start, tsin);
6904         if (ack_type == BBR_CUM_ACKED)
6905                 bbr->rc_ack_is_cumack = 1;
6906         else
6907                 bbr->rc_ack_is_cumack = 0;
6908         old_rttprop = bbr_get_rtt(bbr, BBR_RTT_PROP);
6909         /*
6910          * Note the following code differs to the original
6911          * BBR spec. It calls for <= not <. However after a
6912          * long discussion in email with Neal, he acknowledged
6913          * that it should be < than so that we will have flows
6914          * going into probe-rtt (we were seeing cases where that
6915          * did not happen and caused ugly things to occur). We
6916          * have added this agreed upon fix to our code base.
6917          */
6918         if (rtt < old_rttprop) {
6919                 /* Update when we last saw a rtt drop */
6920                 bbr_log_rtt_shrinks(bbr, cts, 0, rtt, __LINE__, BBR_RTTS_NEWRTT, 0);
6921                 bbr_set_reduced_rtt(bbr, cts, __LINE__);
6922         }
6923         bbr_log_type_bbrrttprop(bbr, rtt, (rsm ? rsm->r_end : 0), uts, cts,
6924             match, rsm->r_start, rsm->r_flags);
6925         apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
6926         if (old_rttprop != bbr_get_rtt(bbr, BBR_RTT_PROP)) {
6927                 /*
6928                  * The RTT-prop moved, reset the target (may be a
6929                  * nop for some states).
6930                  */
6931                 bbr_set_state_target(bbr, __LINE__);
6932                 if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)
6933                         bbr_log_rtt_shrinks(bbr, cts, 0, 0,
6934                                             __LINE__, BBR_RTTS_NEW_TARGET, 0);
6935                 else if (old_rttprop < bbr_get_rtt(bbr, BBR_RTT_PROP))
6936                         /* It went up */
6937                         bbr_check_probe_rtt_limits(bbr, cts);
6938         }
6939         if ((bbr->rc_use_google == 0) &&
6940             (match == BBR_RTT_BY_TIMESTAMP)) {
6941                 /*
6942                  * We don't do b/w update with
6943                  * these since they are not really
6944                  * reliable.
6945                  */
6946                 return;
6947         }
6948         if (bbr->r_ctl.r_app_limited_until &&
6949             (bbr->r_ctl.rc_delivered >= bbr->r_ctl.r_app_limited_until)) {
6950                 /* We are no longer app-limited */
6951                 bbr->r_ctl.r_app_limited_until = 0;
6952         }
6953         if (bbr->rc_use_google) {
6954                 bbr_google_measurement(bbr, rsm, rtt, cts);
6955         } else {
6956                 bbr_nf_measurement(bbr, rsm, rtt, cts);
6957         }
6958 }
6959
6960 /*
6961  * Convert a timestamp that the main stack
6962  * uses (milliseconds) into one that bbr uses
6963  * (microseconds). Return that converted timestamp.
6964  */
6965 static uint32_t
6966 bbr_ts_convert(uint32_t cts) {
6967         uint32_t sec, msec;
6968
6969         sec = cts / MS_IN_USEC;
6970         msec = cts - (MS_IN_USEC * sec);
6971         return ((sec * USECS_IN_SECOND) + (msec * MS_IN_USEC));
6972 }
6973
6974 /*
6975  * Return 0 if we did not update the RTT time, return
6976  * 1 if we did.
6977  */
6978 static int
6979 bbr_update_rtt(struct tcpcb *tp, struct tcp_bbr *bbr,
6980     struct bbr_sendmap *rsm, struct tcpopt *to, uint32_t cts, int32_t ack_type, uint32_t th_ack)
6981 {
6982         int32_t i;
6983         uint32_t t, uts = 0;
6984
6985         if ((rsm->r_flags & BBR_ACKED) ||
6986             (rsm->r_flags & BBR_WAS_RENEGED) ||
6987             (rsm->r_flags & BBR_RXT_CLEARED)) {
6988                 /* Already done */
6989                 return (0);
6990         }
6991         if (rsm->r_rtr_cnt == 1) {
6992                 /*
6993                  * Only one transmit. Hopefully the normal case.
6994                  */
6995                 if (TSTMP_GT(cts, rsm->r_tim_lastsent[0]))
6996                         t = cts - rsm->r_tim_lastsent[0];
6997                 else
6998                         t = 1;
6999                 if ((int)t <= 0)
7000                         t = 1;
7001                 bbr->r_ctl.rc_last_rtt = t;
7002                 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
7003                                     BBR_RTT_BY_EXACTMATCH, rsm->r_tim_lastsent[0], ack_type, to);
7004                 return (1);
7005         }
7006         /* Convert to usecs */
7007         if ((bbr_can_use_ts_for_rtt == 1) &&
7008             (bbr->rc_use_google == 1) &&
7009             (ack_type == BBR_CUM_ACKED) &&
7010             (to->to_flags & TOF_TS) &&
7011             (to->to_tsecr != 0)) {
7012
7013                 t = tcp_tv_to_mssectick(&bbr->rc_tv) - to->to_tsecr;
7014                 if (t < 1)
7015                         t = 1;
7016                 t *= MS_IN_USEC;
7017                 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, 0,
7018                                     BBR_RTT_BY_TIMESTAMP,
7019                                     rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)],
7020                                     ack_type, to);
7021                 return (1);
7022         }
7023         uts = bbr_ts_convert(to->to_tsecr);
7024         if ((to->to_flags & TOF_TS) &&
7025             (to->to_tsecr != 0) &&
7026             (ack_type == BBR_CUM_ACKED) &&
7027             ((rsm->r_flags & BBR_OVERMAX) == 0)) {
7028                 /*
7029                  * Now which timestamp does it match? In this block the ACK
7030                  * may be coming from a previous transmission.
7031                  */
7032                 uint32_t fudge;
7033
7034                 fudge = BBR_TIMER_FUDGE;
7035                 for (i = 0; i < rsm->r_rtr_cnt; i++) {
7036                         if ((SEQ_GEQ(uts, (rsm->r_tim_lastsent[i] - fudge))) &&
7037                             (SEQ_LEQ(uts, (rsm->r_tim_lastsent[i] + fudge)))) {
7038                                 if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
7039                                         t = cts - rsm->r_tim_lastsent[i];
7040                                 else
7041                                         t = 1;
7042                                 if ((int)t <= 0)
7043                                         t = 1;
7044                                 bbr->r_ctl.rc_last_rtt = t;
7045                                 bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_TSMATCHING,
7046                                                     rsm->r_tim_lastsent[i], ack_type, to);
7047                                 if ((i + 1) < rsm->r_rtr_cnt) {
7048                                         /* Likely */
7049                                         bbr_earlier_retran(tp, bbr, rsm, t, cts, ack_type);
7050                                 } else if (rsm->r_flags & BBR_TLP) {
7051                                         bbr->rc_tlp_rtx_out = 0;
7052                                 }
7053                                 return (1);
7054                         }
7055                 }
7056                 /* Fall through if we can't find a matching timestamp */
7057         }
7058         /*
7059          * Ok its a SACK block that we retransmitted. or a windows
7060          * machine without timestamps. We can tell nothing from the
7061          * time-stamp since its not there or the time the peer last
7062          * recieved a segment that moved forward its cum-ack point.
7063          *
7064          * Lets look at the last retransmit and see what we can tell
7065          * (with BBR for space we only keep 2 note we have to keep
7066          * at least 2 so the map can not be condensed more).
7067          */
7068         i = rsm->r_rtr_cnt - 1;
7069         if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
7070                 t = cts - rsm->r_tim_lastsent[i];
7071         else
7072                 goto not_sure;
7073         if (t < bbr->r_ctl.rc_lowest_rtt) {
7074                 /*
7075                  * We retransmitted and the ack came back in less
7076                  * than the smallest rtt we have observed in the
7077                  * windowed rtt. We most likey did an improper
7078                  * retransmit as outlined in 4.2 Step 3 point 2 in
7079                  * the rack-draft.
7080                  *
7081                  * Use the prior transmission to update all the
7082                  * information as long as there is only one prior
7083                  * transmission.
7084                  */
7085                 if ((rsm->r_flags & BBR_OVERMAX) == 0) {
7086 #ifdef BBR_INVARIANTS
7087                         if (rsm->r_rtr_cnt == 1)
7088                                 panic("rsm:%p bbr:%p rsm has overmax and only 1 retranmit flags:%x?", rsm, bbr, rsm->r_flags);
7089 #endif
7090                         i = rsm->r_rtr_cnt - 2;
7091                         if (TSTMP_GT(cts, rsm->r_tim_lastsent[i]))
7092                                 t = cts - rsm->r_tim_lastsent[i];
7093                         else
7094                                 t = 1;
7095                         bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts, BBR_RTT_BY_EARLIER_RET,
7096                                             rsm->r_tim_lastsent[i], ack_type, to);
7097                         bbr_earlier_retran(tp, bbr, rsm, t, cts, ack_type);
7098                 } else {
7099                         /*
7100                          * Too many prior transmissions, just
7101                          * updated BBR delivered
7102                          */
7103 not_sure:
7104                         bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
7105                                             BBR_RTT_BY_SOME_RETRAN, 0, ack_type, to);
7106                 }
7107         } else {
7108                 /*
7109                  * We retransmitted it and the retransmit did the
7110                  * job.
7111                  */
7112                 if (rsm->r_flags & BBR_TLP)
7113                         bbr->rc_tlp_rtx_out = 0;
7114                 if ((rsm->r_flags & BBR_OVERMAX) == 0)
7115                         bbr_update_bbr_info(bbr, rsm, t, cts, to->to_tsecr, uts,
7116                                             BBR_RTT_BY_THIS_RETRAN, 0, ack_type, to);
7117                 else
7118                         bbr_update_bbr_info(bbr, rsm, 0, cts, to->to_tsecr, uts,
7119                                             BBR_RTT_BY_SOME_RETRAN, 0, ack_type, to);
7120                 return (1);
7121         }
7122         return (0);
7123 }
7124
7125 /*
7126  * Mark the SACK_PASSED flag on all entries prior to rsm send wise.
7127  */
7128 static void
7129 bbr_log_sack_passed(struct tcpcb *tp,
7130     struct tcp_bbr *bbr, struct bbr_sendmap *rsm)
7131 {
7132         struct bbr_sendmap *nrsm;
7133
7134         nrsm = rsm;
7135         TAILQ_FOREACH_REVERSE_FROM(nrsm, &bbr->r_ctl.rc_tmap,
7136             bbr_head, r_tnext) {
7137                 if (nrsm == rsm) {
7138                         /* Skip orginal segment he is acked */
7139                         continue;
7140                 }
7141                 if (nrsm->r_flags & BBR_ACKED) {
7142                         /* Skip ack'd segments */
7143                         continue;
7144                 }
7145                 if (nrsm->r_flags & BBR_SACK_PASSED) {
7146                         /*
7147                          * We found one that is already marked
7148                          * passed, we have been here before and
7149                          * so all others below this are marked.
7150                          */
7151                         break;
7152                 }
7153                 BBR_STAT_INC(bbr_sack_passed);
7154                 nrsm->r_flags |= BBR_SACK_PASSED;
7155                 if (((nrsm->r_flags & BBR_MARKED_LOST) == 0) &&
7156                     bbr_is_lost(bbr, nrsm, bbr->r_ctl.rc_rcvtime)) {
7157                         bbr->r_ctl.rc_lost += nrsm->r_end - nrsm->r_start;
7158                         bbr->r_ctl.rc_lost_bytes += nrsm->r_end - nrsm->r_start;
7159                         nrsm->r_flags |= BBR_MARKED_LOST;
7160                 }
7161                 nrsm->r_flags &= ~BBR_WAS_SACKPASS;
7162         }
7163 }
7164
7165 /*
7166  * Returns the number of bytes that were
7167  * newly ack'd by sack blocks.
7168  */
7169 static uint32_t
7170 bbr_proc_sack_blk(struct tcpcb *tp, struct tcp_bbr *bbr, struct sackblk *sack,
7171     struct tcpopt *to, struct bbr_sendmap **prsm, uint32_t cts)
7172 {
7173         int32_t times = 0;
7174         uint32_t start, end, maxseg, changed = 0;
7175         struct bbr_sendmap *rsm, *nrsm;
7176         int32_t used_ref = 1;
7177         uint8_t went_back = 0, went_fwd = 0;
7178
7179         maxseg = tp->t_maxseg - bbr->rc_last_options;
7180         start = sack->start;
7181         end = sack->end;
7182         rsm = *prsm;
7183         if (rsm == NULL)
7184                 used_ref = 0;
7185
7186         /* Do we locate the block behind where we last were? */
7187         if (rsm && SEQ_LT(start, rsm->r_start)) {
7188                 went_back = 1;
7189                 TAILQ_FOREACH_REVERSE_FROM(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
7190                         if (SEQ_GEQ(start, rsm->r_start) &&
7191                             SEQ_LT(start, rsm->r_end)) {
7192                                 goto do_rest_ofb;
7193                         }
7194                 }
7195         }
7196 start_at_beginning:
7197         went_fwd = 1;
7198         /*
7199          * Ok lets locate the block where this guy is fwd from rsm (if its
7200          * set)
7201          */
7202         TAILQ_FOREACH_FROM(rsm, &bbr->r_ctl.rc_map, r_next) {
7203                 if (SEQ_GEQ(start, rsm->r_start) &&
7204                     SEQ_LT(start, rsm->r_end)) {
7205                         break;
7206                 }
7207         }
7208 do_rest_ofb:
7209         if (rsm == NULL) {
7210                 /*
7211                  * This happens when we get duplicate sack blocks with the
7212                  * same end. For example SACK 4: 100 SACK 3: 100 The sort
7213                  * will not change there location so we would just start at
7214                  * the end of the first one and get lost.
7215                  */
7216                 if (tp->t_flags & TF_SENTFIN) {
7217                         /*
7218                          * Check to see if we have not logged the FIN that
7219                          * went out.
7220                          */
7221                         nrsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7222                         if (nrsm && (nrsm->r_end + 1) == tp->snd_max) {
7223                                 /*
7224                                  * Ok we did not get the FIN logged.
7225                                  */
7226                                 nrsm->r_end++;
7227                                 rsm = nrsm;
7228                                 goto do_rest_ofb;
7229                         }
7230                 }
7231                 if (times == 1) {
7232 #ifdef BBR_INVARIANTS
7233                         panic("tp:%p bbr:%p sack:%p to:%p prsm:%p",
7234                             tp, bbr, sack, to, prsm);
7235 #else
7236                         goto out;
7237 #endif
7238                 }
7239                 times++;
7240                 BBR_STAT_INC(bbr_sack_proc_restart);
7241                 rsm = NULL;
7242                 goto start_at_beginning;
7243         }
7244         /* Ok we have an ACK for some piece of rsm */
7245         if (rsm->r_start != start) {
7246                 /*
7247                  * Need to split this in two pieces the before and after.
7248                  */
7249                 if (bbr_sack_mergable(rsm, start, end))
7250                         nrsm = bbr_alloc_full_limit(bbr);
7251                 else
7252                         nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
7253                 if (nrsm == NULL) {
7254                         /* We could not allocate ignore the sack */
7255                         struct sackblk blk;
7256
7257                         blk.start = start;
7258                         blk.end = end;
7259                         sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7260                         goto out;
7261                 }
7262                 bbr_clone_rsm(bbr, nrsm, rsm, start);
7263                 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7264                 if (rsm->r_in_tmap) {
7265                         TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7266                         nrsm->r_in_tmap = 1;
7267                 }
7268                 rsm->r_flags &= (~BBR_HAS_FIN);
7269                 rsm = nrsm;
7270         }
7271         if (SEQ_GEQ(end, rsm->r_end)) {
7272                 /*
7273                  * The end of this block is either beyond this guy or right
7274                  * at this guy.
7275                  */
7276                 if ((rsm->r_flags & BBR_ACKED) == 0) {
7277                         bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7278                         changed += (rsm->r_end - rsm->r_start);
7279                         bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7280                         bbr_log_sack_passed(tp, bbr, rsm);
7281                         if (rsm->r_flags & BBR_MARKED_LOST) {
7282                                 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7283                         }
7284                         /* Is Reordering occuring? */
7285                         if (rsm->r_flags & BBR_SACK_PASSED) {
7286                                 BBR_STAT_INC(bbr_reorder_seen);
7287                                 bbr->r_ctl.rc_reorder_ts = cts;
7288                                 if (rsm->r_flags & BBR_MARKED_LOST) {
7289                                         bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7290                                         if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7291                                                 /* LT sampling also needs adjustment */
7292                                                 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7293                                 }
7294                         }
7295                         rsm->r_flags |= BBR_ACKED;
7296                         rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7297                         if (rsm->r_in_tmap) {
7298                                 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7299                                 rsm->r_in_tmap = 0;
7300                         }
7301                 }
7302                 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7303                 if (end == rsm->r_end) {
7304                         /* This block only - done */
7305                         goto out;
7306                 }
7307                 /* There is more not coverend by this rsm move on */
7308                 start = rsm->r_end;
7309                 nrsm = TAILQ_NEXT(rsm, r_next);
7310                 rsm = nrsm;
7311                 times = 0;
7312                 goto do_rest_ofb;
7313         }
7314         if (rsm->r_flags & BBR_ACKED) {
7315                 /* Been here done that */
7316                 goto out;
7317         }
7318         /* Ok we need to split off this one at the tail */
7319         if (bbr_sack_mergable(rsm, start, end))
7320                 nrsm = bbr_alloc_full_limit(bbr);
7321         else
7322                 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
7323         if (nrsm == NULL) {
7324                 /* failed XXXrrs what can we do but loose the sack info? */
7325                 struct sackblk blk;
7326
7327                 blk.start = start;
7328                 blk.end = end;
7329                 sack_filter_reject(&bbr->r_ctl.bbr_sf, &blk);
7330                 goto out;
7331         }
7332         /* Clone it */
7333         bbr_clone_rsm(bbr, nrsm, rsm, end);
7334         /* The sack block does not cover this guy fully */
7335         rsm->r_flags &= (~BBR_HAS_FIN);
7336         TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
7337         if (rsm->r_in_tmap) {
7338                 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
7339                 nrsm->r_in_tmap = 1;
7340         }
7341         nrsm->r_dupack = 0;
7342         bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_SACKED, 0);
7343         bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_SACKED);
7344         changed += (rsm->r_end - rsm->r_start);
7345         bbr->r_ctl.rc_sacked += (rsm->r_end - rsm->r_start);
7346         bbr_log_sack_passed(tp, bbr, rsm);
7347         /* Is Reordering occuring? */
7348         if (rsm->r_flags & BBR_MARKED_LOST) {
7349                 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7350         }
7351         if (rsm->r_flags & BBR_SACK_PASSED) {
7352                 BBR_STAT_INC(bbr_reorder_seen);
7353                 bbr->r_ctl.rc_reorder_ts = cts;
7354                 if (rsm->r_flags & BBR_MARKED_LOST) {
7355                         bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7356                         if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7357                                 /* LT sampling also needs adjustment */
7358                                 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7359                 }
7360         }
7361         rsm->r_flags &= ~(BBR_TLP|BBR_WAS_RENEGED|BBR_RXT_CLEARED|BBR_MARKED_LOST);
7362         rsm->r_flags |= BBR_ACKED;
7363         if (rsm->r_in_tmap) {
7364                 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7365                 rsm->r_in_tmap = 0;
7366         }
7367 out:
7368         if (rsm && (rsm->r_flags & BBR_ACKED)) {
7369                 /*
7370                  * Now can we merge this newly acked
7371                  * block with either the previous or
7372                  * next block?
7373                  */
7374                 nrsm = TAILQ_NEXT(rsm, r_next);
7375                 if (nrsm &&
7376                     (nrsm->r_flags & BBR_ACKED)) {
7377                         /* yep this and next can be merged */
7378                         rsm = bbr_merge_rsm(bbr, rsm, nrsm);
7379                 }
7380                 /* Now what about the previous? */
7381                 nrsm = TAILQ_PREV(rsm, bbr_head, r_next);
7382                 if (nrsm &&
7383                     (nrsm->r_flags & BBR_ACKED)) {
7384                         /* yep the previous and this can be merged */
7385                         rsm = bbr_merge_rsm(bbr, nrsm, rsm);
7386                 }
7387         }
7388         if (used_ref == 0) {
7389                 BBR_STAT_INC(bbr_sack_proc_all);
7390         } else {
7391                 BBR_STAT_INC(bbr_sack_proc_short);
7392         }
7393         if (went_fwd && went_back) {
7394                 BBR_STAT_INC(bbr_sack_search_both);
7395         } else if (went_fwd) {
7396                 BBR_STAT_INC(bbr_sack_search_fwd);
7397         } else if (went_back) {
7398                 BBR_STAT_INC(bbr_sack_search_back);
7399         }
7400         /* Save off where the next seq is */
7401         if (rsm)
7402                 bbr->r_ctl.rc_sacklast = TAILQ_NEXT(rsm, r_next);
7403         else
7404                 bbr->r_ctl.rc_sacklast = NULL;
7405         *prsm = rsm;
7406         return (changed);
7407 }
7408
7409
7410 static void inline
7411 bbr_peer_reneges(struct tcp_bbr *bbr, struct bbr_sendmap *rsm, tcp_seq th_ack)
7412 {
7413         struct bbr_sendmap *tmap;
7414
7415         BBR_STAT_INC(bbr_reneges_seen);
7416         tmap = NULL;
7417         while (rsm && (rsm->r_flags & BBR_ACKED)) {
7418                 /* Its no longer sacked, mark it so */
7419                 uint32_t oflags;
7420                 bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7421 #ifdef BBR_INVARIANTS
7422                 if (rsm->r_in_tmap) {
7423                         panic("bbr:%p rsm:%p flags:0x%x in tmap?",
7424                             bbr, rsm, rsm->r_flags);
7425                 }
7426 #endif
7427                 oflags = rsm->r_flags;
7428                 if (rsm->r_flags & BBR_MARKED_LOST) {
7429                         bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7430                         bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7431                         if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7432                                 /* LT sampling also needs adjustment */
7433                                 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7434                 }
7435                 rsm->r_flags &= ~(BBR_ACKED | BBR_SACK_PASSED | BBR_WAS_SACKPASS | BBR_MARKED_LOST);
7436                 rsm->r_flags |= BBR_WAS_RENEGED;
7437                 rsm->r_flags |= BBR_RXT_CLEARED;
7438                 bbr_log_type_rsmclear(bbr, bbr->r_ctl.rc_rcvtime, rsm, oflags, __LINE__);
7439                 /* Rebuild it into our tmap */
7440                 if (tmap == NULL) {
7441                         TAILQ_INSERT_HEAD(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7442                         tmap = rsm;
7443                 } else {
7444                         TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, tmap, rsm, r_tnext);
7445                         tmap = rsm;
7446                 }
7447                 tmap->r_in_tmap = 1;
7448                 /*
7449                  * XXXrrs Delivered? Should we do anything here?
7450                  *
7451                  * Of course we don't on a rxt timeout so maybe its ok that
7452                  * we don't?
7453                  *
7454                  * For now lets not.
7455                  */
7456                 rsm = TAILQ_NEXT(rsm, r_next);
7457         }
7458         /*
7459          * Now lets possibly clear the sack filter so we start recognizing
7460          * sacks that cover this area.
7461          */
7462         sack_filter_clear(&bbr->r_ctl.bbr_sf, th_ack);
7463 }
7464
7465 static void
7466 bbr_log_syn(struct tcpcb *tp, struct tcpopt *to)
7467 {
7468         struct tcp_bbr *bbr;
7469         struct bbr_sendmap *rsm;
7470         uint32_t cts;
7471
7472         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7473         cts = bbr->r_ctl.rc_rcvtime;
7474         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7475         if (rsm && (rsm->r_flags & BBR_HAS_SYN)) {
7476                 if ((rsm->r_end - rsm->r_start) <= 1) {
7477                         /* Log out the SYN completely */
7478                         bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7479                         rsm->r_rtr_bytes = 0;
7480                         TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7481                         if (rsm->r_in_tmap) {
7482                                 TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7483                                 rsm->r_in_tmap = 0;
7484                         }
7485                         if (bbr->r_ctl.rc_next == rsm) {
7486                                 /* scoot along the marker */
7487                                 bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7488                         }
7489                         if (to != NULL)
7490                                 bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, 0);
7491                         bbr_free(bbr, rsm);
7492                 } else {
7493                         /* There is more (Fast open)? strip out SYN. */
7494                         rsm->r_flags &= ~BBR_HAS_SYN;
7495                         rsm->r_start++;
7496                 }
7497         }
7498 }
7499
7500 /*
7501  * Returns the number of bytes that were
7502  * acknowledged by SACK blocks.
7503  */
7504
7505 static uint32_t
7506 bbr_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th,
7507     uint32_t *prev_acked)
7508 {
7509         uint32_t changed, last_seq, entered_recovery = 0;
7510         struct tcp_bbr *bbr;
7511         struct bbr_sendmap *rsm;
7512         struct sackblk sack, sack_blocks[TCP_MAX_SACK + 1];
7513         register uint32_t th_ack;
7514         int32_t i, j, k, new_sb, num_sack_blks = 0;
7515         uint32_t cts, acked, ack_point, sack_changed = 0;
7516         uint32_t p_maxseg, maxseg, p_acked = 0;
7517
7518         INP_WLOCK_ASSERT(tp->t_inpcb);
7519         if (th->th_flags & TH_RST) {
7520                 /* We don't log resets */
7521                 return (0);
7522         }
7523         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7524         cts = bbr->r_ctl.rc_rcvtime;
7525
7526         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7527         changed = 0;
7528         maxseg = tp->t_maxseg - bbr->rc_last_options;
7529         p_maxseg = min(bbr->r_ctl.rc_pace_max_segs, maxseg);
7530         th_ack = th->th_ack;
7531         if (SEQ_GT(th_ack, tp->snd_una)) {
7532                 acked = th_ack - tp->snd_una;
7533                 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_UPDATE, __LINE__);
7534                 bbr->rc_tp->t_acktime = ticks;
7535         } else
7536                 acked = 0;
7537         if (SEQ_LEQ(th_ack, tp->snd_una)) {
7538                 /* Only sent here for sack processing */
7539                 goto proc_sack;
7540         }
7541         if (rsm && SEQ_GT(th_ack, rsm->r_start)) {
7542                 changed = th_ack - rsm->r_start;
7543         } else if ((rsm == NULL) && ((th_ack - 1) == tp->iss)) {
7544                 /*
7545                  * For the SYN incoming case we will not have called
7546                  * tcp_output for the sending of the SYN, so there will be
7547                  * no map. All other cases should probably be a panic.
7548                  */
7549                 if ((to->to_flags & TOF_TS) && (to->to_tsecr != 0)) {
7550                         /*
7551                          * We have a timestamp that can be used to generate
7552                          * an initial RTT.
7553                          */
7554                         uint32_t ts, now, rtt;
7555
7556                         ts = bbr_ts_convert(to->to_tsecr);
7557                         now = bbr_ts_convert(tcp_tv_to_mssectick(&bbr->rc_tv));
7558                         rtt = now - ts;
7559                         if (rtt < 1)
7560                                 rtt = 1;
7561                         bbr_log_type_bbrrttprop(bbr, rtt,
7562                                                 tp->iss, 0, cts,
7563                                                 BBR_RTT_BY_TIMESTAMP, tp->iss, 0);
7564                         apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
7565                         changed = 1;
7566                         bbr->r_wanted_output = 1;
7567                         goto out;
7568                 }
7569                 goto proc_sack;
7570         } else if (rsm == NULL) {
7571                 goto out;
7572         }
7573         if (changed) {
7574                 /*
7575                  * The ACK point is advancing to th_ack, we must drop off
7576                  * the packets in the rack log and calculate any eligble
7577                  * RTT's.
7578                  */
7579                 bbr->r_wanted_output = 1;
7580 more:
7581                 if (rsm == NULL) {
7582
7583                         if (tp->t_flags & TF_SENTFIN) {
7584                                 /* if we send a FIN we will not hav a map */
7585                                 goto proc_sack;
7586                         }
7587 #ifdef BBR_INVARIANTS
7588                         panic("No rack map tp:%p for th:%p state:%d bbr:%p snd_una:%u snd_max:%u chg:%d\n",
7589                             tp,
7590                             th, tp->t_state, bbr,
7591                             tp->snd_una, tp->snd_max, changed);
7592 #endif
7593                         goto proc_sack;
7594                 }
7595         }
7596         if (SEQ_LT(th_ack, rsm->r_start)) {
7597                 /* Huh map is missing this */
7598 #ifdef BBR_INVARIANTS
7599                 printf("Rack map starts at r_start:%u for th_ack:%u huh? ts:%d rs:%d bbr:%p\n",
7600                     rsm->r_start,
7601                     th_ack, tp->t_state,
7602                     bbr->r_state, bbr);
7603                 panic("th-ack is bad bbr:%p tp:%p", bbr, tp);
7604 #endif
7605                 goto proc_sack;
7606         } else if (th_ack == rsm->r_start) {
7607                 /* None here to ack */
7608                 goto proc_sack;
7609         }
7610         /*
7611          * Clear the dup ack counter, it will
7612          * either be freed or if there is some
7613          * remaining we need to start it at zero.
7614          */
7615         rsm->r_dupack = 0;
7616         /* Now do we consume the whole thing? */
7617         if (SEQ_GEQ(th_ack, rsm->r_end)) {
7618                 /* Its all consumed. */
7619                 uint32_t left;
7620
7621                 if (rsm->r_flags & BBR_ACKED) {
7622                         /*
7623                          * It was acked on the scoreboard -- remove it from
7624                          * total
7625                          */
7626                         p_acked += (rsm->r_end - rsm->r_start);
7627                         bbr->r_ctl.rc_sacked -= (rsm->r_end - rsm->r_start);
7628                         if (bbr->r_ctl.rc_sacked == 0)
7629                                 bbr->r_ctl.rc_sacklast = NULL;
7630                 } else {
7631                         bbr_update_rtt(tp, bbr, rsm, to, cts, BBR_CUM_ACKED, th_ack);
7632                         if (rsm->r_flags & BBR_MARKED_LOST) {
7633                                 bbr->r_ctl.rc_lost_bytes -= rsm->r_end - rsm->r_start;
7634                         }
7635                         if (rsm->r_flags & BBR_SACK_PASSED) {
7636                                 /*
7637                                  * There are acked segments ACKED on the
7638                                  * scoreboard further up. We are seeing
7639                                  * reordering.
7640                                  */
7641                                 BBR_STAT_INC(bbr_reorder_seen);
7642                                 bbr->r_ctl.rc_reorder_ts = cts;
7643                                 if (rsm->r_flags & BBR_MARKED_LOST) {
7644                                         bbr->r_ctl.rc_lost -= rsm->r_end - rsm->r_start;
7645                                         if (SEQ_GT(bbr->r_ctl.rc_lt_lost, bbr->r_ctl.rc_lost))
7646                                                 /* LT sampling also needs adjustment */
7647                                                 bbr->r_ctl.rc_lt_lost = bbr->r_ctl.rc_lost;
7648                                 }
7649                         }
7650                         rsm->r_flags &= ~BBR_MARKED_LOST;
7651                 }
7652                 bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7653                 rsm->r_rtr_bytes = 0;
7654                 TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
7655                 if (rsm->r_in_tmap) {
7656                         TAILQ_REMOVE(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
7657                         rsm->r_in_tmap = 0;
7658                 }
7659                 if (bbr->r_ctl.rc_next == rsm) {
7660                         /* scoot along the marker */
7661                         bbr->r_ctl.rc_next = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7662                 }
7663                 bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7664                 /* Adjust the packet counts */
7665                 left = th_ack - rsm->r_end;
7666                 /* Free back to zone */
7667                 bbr_free(bbr, rsm);
7668                 if (left) {
7669                         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7670                         goto more;
7671                 }
7672                 goto proc_sack;
7673         }
7674         if (rsm->r_flags & BBR_ACKED) {
7675                 /*
7676                  * It was acked on the scoreboard -- remove it from total
7677                  * for the part being cum-acked.
7678                  */
7679                 p_acked += (rsm->r_end - rsm->r_start);
7680                 bbr->r_ctl.rc_sacked -= (th_ack - rsm->r_start);
7681                 if (bbr->r_ctl.rc_sacked == 0)
7682                         bbr->r_ctl.rc_sacklast = NULL;
7683         } else {
7684                 /*
7685                  * It was acked up to th_ack point for the first time
7686                  */
7687                 struct bbr_sendmap lrsm;
7688
7689                 memcpy(&lrsm, rsm, sizeof(struct bbr_sendmap));
7690                 lrsm.r_end = th_ack;
7691                 bbr_update_rtt(tp, bbr, &lrsm, to, cts, BBR_CUM_ACKED, th_ack);
7692         }
7693         if ((rsm->r_flags & BBR_MARKED_LOST) &&
7694             ((rsm->r_flags & BBR_ACKED) == 0)) {
7695                 /*
7696                  * It was marked lost and partly ack'd now
7697                  * for the first time. We lower the rc_lost_bytes
7698                  * and still leave it MARKED.
7699                  */
7700                 bbr->r_ctl.rc_lost_bytes -= th_ack - rsm->r_start;
7701         }
7702         bbr_isit_a_pkt_epoch(bbr, cts, rsm, __LINE__, BBR_CUM_ACKED);
7703         bbr->r_ctl.rc_holes_rxt -= rsm->r_rtr_bytes;
7704         rsm->r_rtr_bytes = 0;
7705         /* adjust packet count */
7706         rsm->r_start = th_ack;
7707 proc_sack:
7708         /* Check for reneging */
7709         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
7710         if (rsm && (rsm->r_flags & BBR_ACKED) && (th_ack == rsm->r_start)) {
7711                 /*
7712                  * The peer has moved snd_una up to the edge of this send,
7713                  * i.e. one that it had previously acked. The only way that
7714                  * can be true if the peer threw away data (space issues)
7715                  * that it had previously sacked (else it would have given
7716                  * us snd_una up to (rsm->r_end). We need to undo the acked
7717                  * markings here.
7718                  *
7719                  * Note we have to look to make sure th_ack is our
7720                  * rsm->r_start in case we get an old ack where th_ack is
7721                  * behind snd_una.
7722                  */
7723                 bbr_peer_reneges(bbr, rsm, th->th_ack);
7724         }
7725         if ((to->to_flags & TOF_SACK) == 0) {
7726                 /* We are done nothing left to log */
7727                 goto out;
7728         }
7729         rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_map, bbr_sendmap, r_next);
7730         if (rsm) {
7731                 last_seq = rsm->r_end;
7732         } else {
7733                 last_seq = tp->snd_max;
7734         }
7735         /* Sack block processing */
7736         if (SEQ_GT(th_ack, tp->snd_una))
7737                 ack_point = th_ack;
7738         else
7739                 ack_point = tp->snd_una;
7740         for (i = 0; i < to->to_nsacks; i++) {
7741                 bcopy((to->to_sacks + i * TCPOLEN_SACK),
7742                     &sack, sizeof(sack));
7743                 sack.start = ntohl(sack.start);
7744                 sack.end = ntohl(sack.end);
7745                 if (SEQ_GT(sack.end, sack.start) &&
7746                     SEQ_GT(sack.start, ack_point) &&
7747                     SEQ_LT(sack.start, tp->snd_max) &&
7748                     SEQ_GT(sack.end, ack_point) &&
7749                     SEQ_LEQ(sack.end, tp->snd_max)) {
7750                         if ((bbr->r_ctl.rc_num_small_maps_alloced > bbr_sack_block_limit) &&
7751                             (SEQ_LT(sack.end, last_seq)) &&
7752                             ((sack.end - sack.start) < (p_maxseg / 8))) {
7753                                 /*
7754                                  * Not the last piece and its smaller than
7755                                  * 1/8th of a p_maxseg. We ignore this.
7756                                  */
7757                                 BBR_STAT_INC(bbr_runt_sacks);
7758                                 continue;
7759                         }
7760                         sack_blocks[num_sack_blks] = sack;
7761                         num_sack_blks++;
7762 #ifdef NETFLIX_STATS
7763                 } else if (SEQ_LEQ(sack.start, th_ack) &&
7764                     SEQ_LEQ(sack.end, th_ack)) {
7765                         /*
7766                          * Its a D-SACK block.
7767                          */
7768                         tcp_record_dsack(sack.start, sack.end);
7769 #endif
7770                 }
7771         }
7772         if (num_sack_blks == 0)
7773                 goto out;
7774         /*
7775          * Sort the SACK blocks so we can update the rack scoreboard with
7776          * just one pass.
7777          */
7778         new_sb = sack_filter_blks(&bbr->r_ctl.bbr_sf, sack_blocks,
7779                                   num_sack_blks, th->th_ack);
7780         ctf_log_sack_filter(bbr->rc_tp, new_sb, sack_blocks);
7781         BBR_STAT_ADD(bbr_sack_blocks, num_sack_blks);
7782         BBR_STAT_ADD(bbr_sack_blocks_skip, (num_sack_blks - new_sb));
7783         num_sack_blks = new_sb;
7784         if (num_sack_blks < 2) {
7785                 goto do_sack_work;
7786         }
7787         /* Sort the sacks */
7788         for (i = 0; i < num_sack_blks; i++) {
7789                 for (j = i + 1; j < num_sack_blks; j++) {
7790                         if (SEQ_GT(sack_blocks[i].end, sack_blocks[j].end)) {
7791                                 sack = sack_blocks[i];
7792                                 sack_blocks[i] = sack_blocks[j];
7793                                 sack_blocks[j] = sack;
7794                         }
7795                 }
7796         }
7797         /*
7798          * Now are any of the sack block ends the same (yes some
7799          * implememtations send these)?
7800          */
7801 again:
7802         if (num_sack_blks > 1) {
7803                 for (i = 0; i < num_sack_blks; i++) {
7804                         for (j = i + 1; j < num_sack_blks; j++) {
7805                                 if (sack_blocks[i].end == sack_blocks[j].end) {
7806                                         /*
7807                                          * Ok these two have the same end we
7808                                          * want the smallest end and then
7809                                          * throw away the larger and start
7810                                          * again.
7811                                          */
7812                                         if (SEQ_LT(sack_blocks[j].start, sack_blocks[i].start)) {
7813                                                 /*
7814                                                  * The second block covers
7815                                                  * more area use that
7816                                                  */
7817                                                 sack_blocks[i].start = sack_blocks[j].start;
7818                                         }
7819                                         /*
7820                                          * Now collapse out the dup-sack and
7821                                          * lower the count
7822                                          */
7823                                         for (k = (j + 1); k < num_sack_blks; k++) {
7824                                                 sack_blocks[j].start = sack_blocks[k].start;
7825                                                 sack_blocks[j].end = sack_blocks[k].end;
7826                                                 j++;
7827                                         }
7828                                         num_sack_blks--;
7829                                         goto again;
7830                                 }
7831                         }
7832                 }
7833         }
7834 do_sack_work:
7835         rsm = bbr->r_ctl.rc_sacklast;
7836         for (i = 0; i < num_sack_blks; i++) {
7837                 acked = bbr_proc_sack_blk(tp, bbr, &sack_blocks[i], to, &rsm, cts);
7838                 if (acked) {
7839                         bbr->r_wanted_output = 1;
7840                         changed += acked;
7841                         sack_changed += acked;
7842                 }
7843         }
7844 out:
7845         *prev_acked = p_acked;
7846         if ((sack_changed) && (!IN_RECOVERY(tp->t_flags))) {
7847                 /*
7848                  * Ok we have a high probability that we need to go in to
7849                  * recovery since we have data sack'd
7850                  */
7851                 struct bbr_sendmap *rsm;
7852
7853                 rsm = bbr_check_recovery_mode(tp, bbr, cts);
7854                 if (rsm) {
7855                         /* Enter recovery */
7856                         entered_recovery = 1;
7857                         bbr->r_wanted_output = 1;
7858                         /*
7859                          * When we enter recovery we need to assure we send
7860                          * one packet.
7861                          */
7862                         if (bbr->r_ctl.rc_resend == NULL) {
7863                                 bbr->r_ctl.rc_resend = rsm;
7864                         }
7865                 }
7866         }
7867         if (IN_RECOVERY(tp->t_flags) && (entered_recovery == 0)) {
7868                 /*
7869                  * See if we need to rack-retransmit anything if so set it
7870                  * up as the thing to resend assuming something else is not
7871                  * already in that position.
7872                  */
7873                 if (bbr->r_ctl.rc_resend == NULL) {
7874                         bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
7875                 }
7876         }
7877         /*
7878          * We return the amount that changed via sack, this is used by the
7879          * ack-received code to augment what was changed between th_ack <->
7880          * snd_una.
7881          */
7882         return (sack_changed);
7883 }
7884
7885 static void
7886 bbr_strike_dupack(struct tcp_bbr *bbr)
7887 {
7888         struct bbr_sendmap *rsm;
7889
7890         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_tmap);
7891         if (rsm && (rsm->r_dupack < 0xff)) {
7892                 rsm->r_dupack++;
7893                 if (rsm->r_dupack >= DUP_ACK_THRESHOLD)
7894                         bbr->r_wanted_output = 1;
7895         }
7896 }
7897
7898 /*
7899  * Return value of 1, we do not need to call bbr_process_data().
7900  * return value of 0, bbr_process_data can be called.
7901  * For ret_val if its 0 the TCB is locked and valid, if its non-zero
7902  * its unlocked and probably unsafe to touch the TCB.
7903  */
7904 static int
7905 bbr_process_ack(struct mbuf *m, struct tcphdr *th, struct socket *so,
7906     struct tcpcb *tp, struct tcpopt *to,
7907     uint32_t tiwin, int32_t tlen,
7908     int32_t * ofia, int32_t thflags, int32_t * ret_val)
7909 {
7910         int32_t ourfinisacked = 0;
7911         int32_t acked_amount;
7912         uint16_t nsegs;
7913         int32_t acked;
7914         uint32_t lost, sack_changed = 0;
7915         struct mbuf *mfree;
7916         struct tcp_bbr *bbr;
7917         uint32_t prev_acked = 0;
7918
7919         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
7920         lost = bbr->r_ctl.rc_lost;
7921         nsegs = max(1, m->m_pkthdr.lro_nsegs);
7922         if (SEQ_GT(th->th_ack, tp->snd_max)) {
7923                 ctf_do_dropafterack(m, tp, th, thflags, tlen, ret_val);
7924                 bbr->r_wanted_output = 1;
7925                 return (1);
7926         }
7927         if (SEQ_GEQ(th->th_ack, tp->snd_una) || to->to_nsacks) {
7928                 /* Process the ack */
7929                 if (bbr->rc_in_persist)
7930                         tp->t_rxtshift = 0;
7931                 if ((th->th_ack == tp->snd_una) && (tiwin == tp->snd_wnd))
7932                         bbr_strike_dupack(bbr);
7933                 sack_changed = bbr_log_ack(tp, to, th, &prev_acked);
7934         }
7935         bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, (bbr->r_ctl.rc_lost > lost));
7936         if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
7937                 /*
7938                  * Old ack, behind the last one rcv'd or a duplicate ack
7939                  * with SACK info.
7940                  */
7941                 if (th->th_ack == tp->snd_una) {
7942                         bbr_ack_received(tp, bbr, th, 0, sack_changed, prev_acked, __LINE__, 0);
7943                         if (bbr->r_state == TCPS_SYN_SENT) {
7944                                 /*
7945                                  * Special case on where we sent SYN. When
7946                                  * the SYN-ACK is processed in syn_sent
7947                                  * state it bumps the snd_una. This causes
7948                                  * us to hit here even though we did ack 1
7949                                  * byte.
7950                                  *
7951                                  * Go through the nothing left case so we
7952                                  * send data.
7953                                  */
7954                                 goto nothing_left;
7955                         }
7956                 }
7957                 return (0);
7958         }
7959         /*
7960          * If we reach this point, ACK is not a duplicate, i.e., it ACKs
7961          * something we sent.
7962          */
7963         if (tp->t_flags & TF_NEEDSYN) {
7964                 /*
7965                  * T/TCP: Connection was half-synchronized, and our SYN has
7966                  * been ACK'd (so connection is now fully synchronized).  Go
7967                  * to non-starred state, increment snd_una for ACK of SYN,
7968                  * and check if we can do window scaling.
7969                  */
7970                 tp->t_flags &= ~TF_NEEDSYN;
7971                 tp->snd_una++;
7972                 /* Do window scaling? */
7973                 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
7974                     (TF_RCVD_SCALE | TF_REQ_SCALE)) {
7975                         tp->rcv_scale = tp->request_r_scale;
7976                         /* Send window already scaled. */
7977                 }
7978         }
7979         INP_WLOCK_ASSERT(tp->t_inpcb);
7980
7981         acked = BYTES_THIS_ACK(tp, th);
7982         KMOD_TCPSTAT_ADD(tcps_rcvackpack, (int)nsegs);
7983         KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
7984
7985         /*
7986          * If we just performed our first retransmit, and the ACK arrives
7987          * within our recovery window, then it was a mistake to do the
7988          * retransmit in the first place.  Recover our original cwnd and
7989          * ssthresh, and proceed to transmit where we left off.
7990          */
7991         if (tp->t_flags & TF_PREVVALID) {
7992                 tp->t_flags &= ~TF_PREVVALID;
7993                 if (tp->t_rxtshift == 1 &&
7994                     (int)(ticks - tp->t_badrxtwin) < 0)
7995                         bbr_cong_signal(tp, th, CC_RTO_ERR, NULL);
7996         }
7997         SOCKBUF_LOCK(&so->so_snd);
7998         acked_amount = min(acked, (int)sbavail(&so->so_snd));
7999         tp->snd_wnd -= acked_amount;
8000         mfree = sbcut_locked(&so->so_snd, acked_amount);
8001         /* NB: sowwakeup_locked() does an implicit unlock. */
8002         sowwakeup_locked(so);
8003         m_freem(mfree);
8004         if (SEQ_GT(th->th_ack, tp->snd_una)) {
8005                 bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp));
8006         }
8007         tp->snd_una = th->th_ack;
8008         bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, (bbr->r_ctl.rc_lost - lost));
8009         if (IN_RECOVERY(tp->t_flags)) {
8010                 if (SEQ_LT(th->th_ack, tp->snd_recover) &&
8011                     (SEQ_LT(th->th_ack, tp->snd_max))) {
8012                         tcp_bbr_partialack(tp);
8013                 } else {
8014                         bbr_post_recovery(tp);
8015                 }
8016         }
8017         if (SEQ_GT(tp->snd_una, tp->snd_recover)) {
8018                 tp->snd_recover = tp->snd_una;
8019         }
8020         if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
8021                 tp->snd_nxt = tp->snd_max;
8022         }
8023         if (tp->snd_una == tp->snd_max) {
8024                 /* Nothing left outstanding */
8025 nothing_left:
8026                 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__);
8027                 if (sbavail(&tp->t_inpcb->inp_socket->so_snd) == 0)
8028                         bbr->rc_tp->t_acktime = 0;
8029                 if ((sbused(&so->so_snd) == 0) &&
8030                     (tp->t_flags & TF_SENTFIN)) {
8031                         ourfinisacked = 1;
8032                 }
8033                 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8034                 if (bbr->rc_in_persist == 0) {
8035                         bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
8036                 }
8037                 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
8038                 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
8039                 /*
8040                  * We invalidate the last ack here since we
8041                  * don't want to transfer forward the time
8042                  * for our sum's calculations.
8043                  */
8044                 if ((tp->t_state >= TCPS_FIN_WAIT_1) &&
8045                     (sbavail(&so->so_snd) == 0) &&
8046                     (tp->t_flags2 & TF2_DROP_AF_DATA)) {
8047                         /*
8048                          * The socket was gone and the peer sent data, time
8049                          * to reset him.
8050                          */
8051                         *ret_val = 1;
8052                         tp = tcp_close(tp);
8053                         ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, tlen);
8054                         BBR_STAT_INC(bbr_dropped_af_data);
8055                         return (1);
8056                 }
8057                 /* Set need output so persist might get set */
8058                 bbr->r_wanted_output = 1;
8059         }
8060         if (ofia)
8061                 *ofia = ourfinisacked;
8062         return (0);
8063 }
8064
8065 static void
8066 bbr_enter_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
8067 {
8068         if (bbr->rc_in_persist == 0) {
8069                 bbr_timer_cancel(bbr, __LINE__, cts);
8070                 bbr->r_ctl.rc_last_delay_val = 0;
8071                 tp->t_rxtshift = 0;
8072                 bbr->rc_in_persist = 1;
8073                 bbr->r_ctl.rc_went_idle_time = cts;
8074                 /* We should be capped when rw went to 0 but just in case */
8075                 bbr_log_type_pesist(bbr, cts, 0, line, 1);
8076                 /* Time freezes for the state, so do the accounting now */
8077                 if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
8078                         uint32_t time_in;
8079
8080                         time_in = cts - bbr->r_ctl.rc_bbr_state_time;
8081                         if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
8082                                 int32_t idx;
8083
8084                                 idx = bbr_state_val(bbr);
8085                                 counter_u64_add(bbr_state_time[(idx + 5)], time_in);
8086                         } else {
8087                                 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
8088                         }
8089                 }
8090                 bbr->r_ctl.rc_bbr_state_time = cts;
8091         }
8092 }
8093
8094 static void
8095 bbr_restart_after_idle(struct tcp_bbr *bbr, uint32_t cts, uint32_t idle_time)
8096 {
8097         /*
8098          * Note that if idle time does not exceed our
8099          * threshold, we do nothing continuing the state
8100          * transitions we were last walking through.
8101          */
8102         if (idle_time >= bbr_idle_restart_threshold) {
8103                 if (bbr->rc_use_idle_restart) {
8104                         bbr->rc_bbr_state = BBR_STATE_IDLE_EXIT;
8105                         /*
8106                          * Set our target using BBR_UNIT, so
8107                          * we increase at a dramatic rate but
8108                          * we stop when we get the pipe
8109                          * full again for our current b/w estimate.
8110                          */
8111                         bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
8112                         bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
8113                         bbr_set_state_target(bbr, __LINE__);
8114                         /* Now setup our gains to ramp up */
8115                         bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
8116                         bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
8117                         bbr_log_type_statechange(bbr, cts, __LINE__);
8118                 } else {
8119                         bbr_substate_change(bbr, cts, __LINE__, 1);
8120                 }
8121         }
8122 }
8123
8124 static void
8125 bbr_exit_persist(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts, int32_t line)
8126 {
8127         uint32_t idle_time;
8128
8129         if (bbr->rc_in_persist == 0)
8130                 return;
8131         idle_time = bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time);
8132         bbr->rc_in_persist = 0;
8133         bbr->rc_hit_state_1 = 0;
8134         tp->t_flags &= ~TF_FORCEDATA;
8135         bbr->r_ctl.rc_del_time = cts;
8136         /*
8137          * We invalidate the last ack here since we
8138          * don't want to transfer forward the time
8139          * for our sum's calculations.
8140          */
8141         if (bbr->rc_inp->inp_in_hpts) {
8142                 tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
8143                 bbr->rc_timer_first = 0;
8144                 bbr->r_ctl.rc_hpts_flags = 0;
8145                 bbr->r_ctl.rc_last_delay_val = 0;
8146                 bbr->r_ctl.rc_hptsi_agg_delay = 0;
8147                 bbr->r_agg_early_set = 0;
8148                 bbr->r_ctl.rc_agg_early = 0;
8149         }
8150         bbr_log_type_pesist(bbr, cts, idle_time, line, 0);
8151         if (idle_time >= bbr_rtt_probe_time) {
8152                 /*
8153                  * This qualifies as a RTT_PROBE session since we drop the
8154                  * data outstanding to nothing and waited more than
8155                  * bbr_rtt_probe_time.
8156                  */
8157                 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_PERSIST, 0);
8158                 bbr->r_ctl.last_in_probertt = bbr->r_ctl.rc_rtt_shrinks = cts;
8159         }
8160         tp->t_rxtshift = 0;
8161         /*
8162          * If in probeBW and we have persisted more than an RTT lets do
8163          * special handling.
8164          */
8165         /* Force a time based epoch */
8166         bbr_set_epoch(bbr, cts, __LINE__);
8167         /*
8168          * Setup the lost so we don't count anything against the guy
8169          * we have been stuck with during persists.
8170          */
8171         bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
8172         /* Time un-freezes for the state */
8173         bbr->r_ctl.rc_bbr_state_time = cts;
8174         if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) ||
8175             (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT)) {
8176                 /*
8177                  * If we are going back to probe-bw
8178                  * or probe_rtt, we may need to possibly
8179                  * do a fast restart.
8180                  */
8181                 bbr_restart_after_idle(bbr, cts, idle_time);
8182         }
8183 }
8184
8185 static void
8186 bbr_collapsed_window(struct tcp_bbr *bbr)
8187 {
8188         /*
8189          * Now we must walk the
8190          * send map and divide the
8191          * ones left stranded. These
8192          * guys can't cause us to abort
8193          * the connection and are really
8194          * "unsent". However if a buggy
8195          * client actually did keep some
8196          * of the data i.e. collapsed the win
8197          * and refused to ack and then opened
8198          * the win and acked that data. We would
8199          * get into an ack war, the simplier
8200          * method then of just pretending we
8201          * did not send those segments something
8202          * won't work.
8203          */
8204         struct bbr_sendmap *rsm, *nrsm;
8205         tcp_seq max_seq;
8206         uint32_t maxseg;
8207         int can_split = 0;
8208         int fnd = 0;
8209
8210         maxseg = bbr->rc_tp->t_maxseg - bbr->rc_last_options;
8211         max_seq = bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd;
8212         bbr_log_type_rwnd_collapse(bbr, max_seq, 1, 0);
8213         TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
8214                 /* Find the first seq past or at maxseq */
8215                 if (rsm->r_flags & BBR_RWND_COLLAPSED)
8216                         rsm->r_flags &= ~BBR_RWND_COLLAPSED;
8217                 if (SEQ_GEQ(max_seq, rsm->r_start) &&
8218                     SEQ_GEQ(rsm->r_end, max_seq)) {
8219                         fnd = 1;
8220                         break;
8221                 }
8222         }
8223         bbr->rc_has_collapsed = 0;
8224         if (!fnd) {
8225                 /* Nothing to do strange */
8226                 return;
8227         }
8228         /*
8229          * Now can we split?
8230          *
8231          * We don't want to split if splitting
8232          * would generate too many small segments
8233          * less we let an attacker fragment our
8234          * send_map and leave us out of memory.
8235          */
8236         if ((max_seq != rsm->r_start) &&
8237             (max_seq != rsm->r_end)){
8238                 /* can we split? */
8239                 int res1, res2;
8240
8241                 res1 = max_seq - rsm->r_start;
8242                 res2 = rsm->r_end - max_seq;
8243                 if ((res1 >= (maxseg/8)) &&
8244                     (res2 >= (maxseg/8))) {
8245                         /* No small pieces here */
8246                         can_split = 1;
8247                 } else if (bbr->r_ctl.rc_num_small_maps_alloced < bbr_sack_block_limit) {
8248                         /* We are under the limit */
8249                         can_split = 1;
8250                 }
8251         }
8252         /* Ok do we need to split this rsm? */
8253         if (max_seq == rsm->r_start) {
8254                 /* It's this guy no split required */
8255                 nrsm = rsm;
8256         } else if (max_seq == rsm->r_end) {
8257                 /* It's the next one no split required. */
8258                 nrsm = TAILQ_NEXT(rsm, r_next);
8259                 if (nrsm == NULL) {
8260                         /* Huh? */
8261                         return;
8262                 }
8263         } else if (can_split && SEQ_LT(max_seq, rsm->r_end)) {
8264                 /* yep we need to split it */
8265                 nrsm = bbr_alloc_limit(bbr, BBR_LIMIT_TYPE_SPLIT);
8266                 if (nrsm == NULL) {
8267                         /* failed XXXrrs what can we do mark the whole? */
8268                         nrsm = rsm;
8269                         goto no_split;
8270                 }
8271                 /* Clone it */
8272                 bbr_log_type_rwnd_collapse(bbr, max_seq, 3, 0);
8273                 bbr_clone_rsm(bbr, nrsm, rsm, max_seq);
8274                 TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_map, rsm, nrsm, r_next);
8275                 if (rsm->r_in_tmap) {
8276                         TAILQ_INSERT_AFTER(&bbr->r_ctl.rc_tmap, rsm, nrsm, r_tnext);
8277                         nrsm->r_in_tmap = 1;
8278                 }
8279         } else {
8280                 /*
8281                  * Split not allowed just start here just
8282                  * use this guy.
8283                  */
8284                 nrsm = rsm;
8285         }
8286 no_split:
8287         BBR_STAT_INC(bbr_collapsed_win);
8288         /* reuse fnd as a count */
8289         fnd = 0;
8290         TAILQ_FOREACH_FROM(nrsm, &bbr->r_ctl.rc_map, r_next) {
8291                 nrsm->r_flags |= BBR_RWND_COLLAPSED;
8292                 fnd++;
8293                 bbr->rc_has_collapsed = 1;
8294         }
8295         bbr_log_type_rwnd_collapse(bbr, max_seq, 4, fnd);
8296 }
8297
8298 static void
8299 bbr_un_collapse_window(struct tcp_bbr *bbr)
8300 {
8301         struct bbr_sendmap *rsm;
8302         int cleared = 0;
8303
8304         TAILQ_FOREACH_REVERSE(rsm, &bbr->r_ctl.rc_map, bbr_head, r_next) {
8305                 if (rsm->r_flags & BBR_RWND_COLLAPSED) {
8306                         /* Clear the flag */
8307                         rsm->r_flags &= ~BBR_RWND_COLLAPSED;
8308                         cleared++;
8309                 } else
8310                         break;
8311         }
8312         bbr_log_type_rwnd_collapse(bbr,
8313                                    (bbr->rc_tp->snd_una + bbr->rc_tp->snd_wnd), 0, cleared);
8314         bbr->rc_has_collapsed = 0;
8315 }
8316
8317 /*
8318  * Return value of 1, the TCB is unlocked and most
8319  * likely gone, return value of 0, the TCB is still
8320  * locked.
8321  */
8322 static int
8323 bbr_process_data(struct mbuf *m, struct tcphdr *th, struct socket *so,
8324     struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen,
8325     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
8326 {
8327         /*
8328          * Update window information. Don't look at window if no ACK: TAC's
8329          * send garbage on first SYN.
8330          */
8331         uint16_t nsegs;
8332         int32_t tfo_syn;
8333         struct tcp_bbr *bbr;
8334
8335         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8336         INP_WLOCK_ASSERT(tp->t_inpcb);
8337         nsegs = max(1, m->m_pkthdr.lro_nsegs);
8338         if ((thflags & TH_ACK) &&
8339             (SEQ_LT(tp->snd_wl1, th->th_seq) ||
8340             (tp->snd_wl1 == th->th_seq && (SEQ_LT(tp->snd_wl2, th->th_ack) ||
8341             (tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd))))) {
8342                 /* keep track of pure window updates */
8343                 if (tlen == 0 &&
8344                     tp->snd_wl2 == th->th_ack && tiwin > tp->snd_wnd)
8345                         KMOD_TCPSTAT_INC(tcps_rcvwinupd);
8346                 tp->snd_wnd = tiwin;
8347                 tp->snd_wl1 = th->th_seq;
8348                 tp->snd_wl2 = th->th_ack;
8349                 if (tp->snd_wnd > tp->max_sndwnd)
8350                         tp->max_sndwnd = tp->snd_wnd;
8351                 bbr->r_wanted_output = 1;
8352         } else if (thflags & TH_ACK) {
8353                 if ((tp->snd_wl2 == th->th_ack) && (tiwin < tp->snd_wnd)) {
8354                         tp->snd_wnd = tiwin;
8355                         tp->snd_wl1 = th->th_seq;
8356                         tp->snd_wl2 = th->th_ack;
8357                 }
8358         }
8359         if (tp->snd_wnd < ctf_outstanding(tp))
8360                 /* The peer collapsed its window on us */
8361                 bbr_collapsed_window(bbr);
8362         else if (bbr->rc_has_collapsed)
8363                 bbr_un_collapse_window(bbr);
8364         /* Was persist timer active and now we have window space? */
8365         if ((bbr->rc_in_persist != 0) &&
8366             (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8367                                 bbr_minseg(bbr)))) {
8368                 /*
8369                  * Make the rate persist at end of persist mode if idle long
8370                  * enough
8371                  */
8372                 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8373
8374                 /* Make sure we output to start the timer */
8375                 bbr->r_wanted_output = 1;
8376         }
8377         /* Do we need to enter persist? */
8378         if ((bbr->rc_in_persist == 0) &&
8379             (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8380             TCPS_HAVEESTABLISHED(tp->t_state) &&
8381             (tp->snd_max == tp->snd_una) &&
8382             sbavail(&tp->t_inpcb->inp_socket->so_snd) &&
8383             (sbavail(&tp->t_inpcb->inp_socket->so_snd) > tp->snd_wnd)) {
8384                 /* No send window.. we must enter persist */
8385                 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8386         }
8387         if (tp->t_flags2 & TF2_DROP_AF_DATA) {
8388                 m_freem(m);
8389                 return (0);
8390         }
8391         /*
8392          * Process segments with URG.
8393          */
8394         if ((thflags & TH_URG) && th->th_urp &&
8395             TCPS_HAVERCVDFIN(tp->t_state) == 0) {
8396                 /*
8397                  * This is a kludge, but if we receive and accept random
8398                  * urgent pointers, we'll crash in soreceive.  It's hard to
8399                  * imagine someone actually wanting to send this much urgent
8400                  * data.
8401                  */
8402                 SOCKBUF_LOCK(&so->so_rcv);
8403                 if (th->th_urp + sbavail(&so->so_rcv) > sb_max) {
8404                         th->th_urp = 0; /* XXX */
8405                         thflags &= ~TH_URG;     /* XXX */
8406                         SOCKBUF_UNLOCK(&so->so_rcv);    /* XXX */
8407                         goto dodata;    /* XXX */
8408                 }
8409                 /*
8410                  * If this segment advances the known urgent pointer, then
8411                  * mark the data stream.  This should not happen in
8412                  * CLOSE_WAIT, CLOSING, LAST_ACK or TIME_WAIT STATES since a
8413                  * FIN has been received from the remote side. In these
8414                  * states we ignore the URG.
8415                  *
8416                  * According to RFC961 (Assigned Protocols), the urgent
8417                  * pointer points to the last octet of urgent data.  We
8418                  * continue, however, to consider it to indicate the first
8419                  * octet of data past the urgent section as the original
8420                  * spec states (in one of two places).
8421                  */
8422                 if (SEQ_GT(th->th_seq + th->th_urp, tp->rcv_up)) {
8423                         tp->rcv_up = th->th_seq + th->th_urp;
8424                         so->so_oobmark = sbavail(&so->so_rcv) +
8425                             (tp->rcv_up - tp->rcv_nxt) - 1;
8426                         if (so->so_oobmark == 0)
8427                                 so->so_rcv.sb_state |= SBS_RCVATMARK;
8428                         sohasoutofband(so);
8429                         tp->t_oobflags &= ~(TCPOOB_HAVEDATA | TCPOOB_HADDATA);
8430                 }
8431                 SOCKBUF_UNLOCK(&so->so_rcv);
8432                 /*
8433                  * Remove out of band data so doesn't get presented to user.
8434                  * This can happen independent of advancing the URG pointer,
8435                  * but if two URG's are pending at once, some out-of-band
8436                  * data may creep in... ick.
8437                  */
8438                 if (th->th_urp <= (uint32_t)tlen &&
8439                     !(so->so_options & SO_OOBINLINE)) {
8440                         /* hdr drop is delayed */
8441                         tcp_pulloutofband(so, th, m, drop_hdrlen);
8442                 }
8443         } else {
8444                 /*
8445                  * If no out of band data is expected, pull receive urgent
8446                  * pointer along with the receive window.
8447                  */
8448                 if (SEQ_GT(tp->rcv_nxt, tp->rcv_up))
8449                         tp->rcv_up = tp->rcv_nxt;
8450         }
8451 dodata:                         /* XXX */
8452         INP_WLOCK_ASSERT(tp->t_inpcb);
8453
8454         /*
8455          * Process the segment text, merging it into the TCP sequencing
8456          * queue, and arranging for acknowledgment of receipt if necessary.
8457          * This process logically involves adjusting tp->rcv_wnd as data is
8458          * presented to the user (this happens in tcp_usrreq.c, case
8459          * PRU_RCVD).  If a FIN has already been received on this connection
8460          * then we just ignore the text.
8461          */
8462         tfo_syn = ((tp->t_state == TCPS_SYN_RECEIVED) &&
8463                    IS_FASTOPEN(tp->t_flags));
8464         if ((tlen || (thflags & TH_FIN) || tfo_syn) &&
8465             TCPS_HAVERCVDFIN(tp->t_state) == 0) {
8466                 tcp_seq save_start = th->th_seq;
8467                 tcp_seq save_rnxt  = tp->rcv_nxt;
8468                 int     save_tlen  = tlen;
8469
8470                 m_adj(m, drop_hdrlen);  /* delayed header drop */
8471                 /*
8472                  * Insert segment which includes th into TCP reassembly
8473                  * queue with control block tp.  Set thflags to whether
8474                  * reassembly now includes a segment with FIN.  This handles
8475                  * the common case inline (segment is the next to be
8476                  * received on an established connection, and the queue is
8477                  * empty), avoiding linkage into and removal from the queue
8478                  * and repetition of various conversions. Set DELACK for
8479                  * segments received in order, but ack immediately when
8480                  * segments are out of order (so fast retransmit can work).
8481                  */
8482                 if (th->th_seq == tp->rcv_nxt &&
8483                     SEGQ_EMPTY(tp) &&
8484                     (TCPS_HAVEESTABLISHED(tp->t_state) ||
8485                     tfo_syn)) {
8486 #ifdef NETFLIX_SB_LIMITS
8487                         u_int mcnt, appended;
8488
8489                         if (so->so_rcv.sb_shlim) {
8490                                 mcnt = m_memcnt(m);
8491                                 appended = 0;
8492                                 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
8493                                     CFO_NOSLEEP, NULL) == false) {
8494                                         counter_u64_add(tcp_sb_shlim_fails, 1);
8495                                         m_freem(m);
8496                                         return (0);
8497                                 }
8498                         }
8499
8500 #endif
8501                         if (DELAY_ACK(tp, bbr, nsegs) || tfo_syn) {
8502                                 bbr->bbr_segs_rcvd += max(1, nsegs);
8503                                 tp->t_flags |= TF_DELACK;
8504                                 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8505                         } else {
8506                                 bbr->r_wanted_output = 1;
8507                                 tp->t_flags |= TF_ACKNOW;
8508                         }
8509                         tp->rcv_nxt += tlen;
8510                         thflags = th->th_flags & TH_FIN;
8511                         KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs);
8512                         KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
8513                         SOCKBUF_LOCK(&so->so_rcv);
8514                         if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
8515                                 m_freem(m);
8516                         else
8517 #ifdef NETFLIX_SB_LIMITS
8518                                 appended =
8519 #endif
8520                                         sbappendstream_locked(&so->so_rcv, m, 0);
8521                         /* NB: sorwakeup_locked() does an implicit unlock. */
8522                         sorwakeup_locked(so);
8523 #ifdef NETFLIX_SB_LIMITS
8524                         if (so->so_rcv.sb_shlim && appended != mcnt)
8525                                 counter_fo_release(so->so_rcv.sb_shlim,
8526                                     mcnt - appended);
8527 #endif
8528                 } else {
8529                         /*
8530                          * XXX: Due to the header drop above "th" is
8531                          * theoretically invalid by now.  Fortunately
8532                          * m_adj() doesn't actually frees any mbufs when
8533                          * trimming from the head.
8534                          */
8535                         tcp_seq temp = save_start;
8536                         thflags = tcp_reass(tp, th, &temp, &tlen, m);
8537                         tp->t_flags |= TF_ACKNOW;
8538                 }
8539                 if ((tp->t_flags & TF_SACK_PERMIT) && (save_tlen > 0)) {
8540                         if ((tlen == 0) && (SEQ_LT(save_start, save_rnxt))) {
8541                                 /*
8542                                  * DSACK actually handled in the fastpath
8543                                  * above.
8544                                  */
8545                                 tcp_update_sack_list(tp, save_start,
8546                                     save_start + save_tlen);
8547                         } else if ((tlen > 0) && SEQ_GT(tp->rcv_nxt, save_rnxt)) {
8548                                 if ((tp->rcv_numsacks >= 1) &&
8549                                     (tp->sackblks[0].end == save_start)) {
8550                                         /*
8551                                          * Partial overlap, recorded at todrop
8552                                          * above.
8553                                          */
8554                                         tcp_update_sack_list(tp,
8555                                             tp->sackblks[0].start,
8556                                             tp->sackblks[0].end);
8557                                 } else {
8558                                         tcp_update_dsack_list(tp, save_start,
8559                                             save_start + save_tlen);
8560                                 }
8561                         } else if (tlen >= save_tlen) {
8562                                 /* Update of sackblks. */
8563                                 tcp_update_dsack_list(tp, save_start,
8564                                     save_start + save_tlen);
8565                         } else if (tlen > 0) {
8566                                 tcp_update_dsack_list(tp, save_start,
8567                                     save_start + tlen);
8568                         }
8569                 }
8570         } else {
8571                 m_freem(m);
8572                 thflags &= ~TH_FIN;
8573         }
8574
8575         /*
8576          * If FIN is received ACK the FIN and let the user know that the
8577          * connection is closing.
8578          */
8579         if (thflags & TH_FIN) {
8580                 if (TCPS_HAVERCVDFIN(tp->t_state) == 0) {
8581                         socantrcvmore(so);
8582                         /*
8583                          * If connection is half-synchronized (ie NEEDSYN
8584                          * flag on) then delay ACK, so it may be piggybacked
8585                          * when SYN is sent. Otherwise, since we received a
8586                          * FIN then no more input can be expected, send ACK
8587                          * now.
8588                          */
8589                         if (tp->t_flags & TF_NEEDSYN) {
8590                                 tp->t_flags |= TF_DELACK;
8591                                 bbr_timer_cancel(bbr,
8592                                     __LINE__, bbr->r_ctl.rc_rcvtime);
8593                         } else {
8594                                 tp->t_flags |= TF_ACKNOW;
8595                         }
8596                         tp->rcv_nxt++;
8597                 }
8598                 switch (tp->t_state) {
8599
8600                         /*
8601                          * In SYN_RECEIVED and ESTABLISHED STATES enter the
8602                          * CLOSE_WAIT state.
8603                          */
8604                 case TCPS_SYN_RECEIVED:
8605                         tp->t_starttime = ticks;
8606                         /* FALLTHROUGH */
8607                 case TCPS_ESTABLISHED:
8608                         tcp_state_change(tp, TCPS_CLOSE_WAIT);
8609                         break;
8610
8611                         /*
8612                          * If still in FIN_WAIT_1 STATE FIN has not been
8613                          * acked so enter the CLOSING state.
8614                          */
8615                 case TCPS_FIN_WAIT_1:
8616                         tcp_state_change(tp, TCPS_CLOSING);
8617                         break;
8618
8619                         /*
8620                          * In FIN_WAIT_2 state enter the TIME_WAIT state,
8621                          * starting the time-wait timer, turning off the
8622                          * other standard timers.
8623                          */
8624                 case TCPS_FIN_WAIT_2:
8625                         bbr->rc_timer_first = 1;
8626                         bbr_timer_cancel(bbr,
8627                             __LINE__, bbr->r_ctl.rc_rcvtime);
8628                         INP_WLOCK_ASSERT(tp->t_inpcb);
8629                         tcp_twstart(tp);
8630                         return (1);
8631                 }
8632         }
8633         /*
8634          * Return any desired output.
8635          */
8636         if ((tp->t_flags & TF_ACKNOW) ||
8637             (sbavail(&so->so_snd) > ctf_outstanding(tp))) {
8638                 bbr->r_wanted_output = 1;
8639         }
8640         INP_WLOCK_ASSERT(tp->t_inpcb);
8641         return (0);
8642 }
8643
8644 /*
8645  * Here nothing is really faster, its just that we
8646  * have broken out the fast-data path also just like
8647  * the fast-ack. Return 1 if we processed the packet
8648  * return 0 if you need to take the "slow-path".
8649  */
8650 static int
8651 bbr_do_fastnewdata(struct mbuf *m, struct tcphdr *th, struct socket *so,
8652     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8653     uint32_t tiwin, int32_t nxt_pkt)
8654 {
8655         uint16_t nsegs;
8656         int32_t newsize = 0;    /* automatic sockbuf scaling */
8657         struct tcp_bbr *bbr;
8658 #ifdef NETFLIX_SB_LIMITS
8659         u_int mcnt, appended;
8660 #endif
8661 #ifdef TCPDEBUG
8662         /*
8663          * The size of tcp_saveipgen must be the size of the max ip header,
8664          * now IPv6.
8665          */
8666         u_char tcp_saveipgen[IP6_HDR_LEN];
8667         struct tcphdr tcp_savetcp;
8668         short ostate = 0;
8669
8670 #endif
8671         /* On the hpts and we would have called output */
8672         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8673
8674         /*
8675          * If last ACK falls within this segment's sequence numbers, record
8676          * the timestamp. NOTE that the test is modified according to the
8677          * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
8678          */
8679         if (bbr->r_ctl.rc_resend != NULL) {
8680                 return (0);
8681         }
8682         if (tiwin && tiwin != tp->snd_wnd) {
8683                 return (0);
8684         }
8685         if (__predict_false((tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN)))) {
8686                 return (0);
8687         }
8688         if (__predict_false((to->to_flags & TOF_TS) &&
8689             (TSTMP_LT(to->to_tsval, tp->ts_recent)))) {
8690                 return (0);
8691         }
8692         if (__predict_false((th->th_ack != tp->snd_una))) {
8693                 return (0);
8694         }
8695         if (__predict_false(tlen > sbspace(&so->so_rcv))) {
8696                 return (0);
8697         }
8698         if ((to->to_flags & TOF_TS) != 0 &&
8699             SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
8700                 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
8701                 tp->ts_recent = to->to_tsval;
8702         }
8703         /*
8704          * This is a pure, in-sequence data packet with nothing on the
8705          * reassembly queue and we have enough buffer space to take it.
8706          */
8707         nsegs = max(1, m->m_pkthdr.lro_nsegs);
8708
8709 #ifdef NETFLIX_SB_LIMITS
8710         if (so->so_rcv.sb_shlim) {
8711                 mcnt = m_memcnt(m);
8712                 appended = 0;
8713                 if (counter_fo_get(so->so_rcv.sb_shlim, mcnt,
8714                     CFO_NOSLEEP, NULL) == false) {
8715                         counter_u64_add(tcp_sb_shlim_fails, 1);
8716                         m_freem(m);
8717                         return (1);
8718                 }
8719         }
8720 #endif
8721         /* Clean receiver SACK report if present */
8722         if (tp->rcv_numsacks)
8723                 tcp_clean_sackreport(tp);
8724         KMOD_TCPSTAT_INC(tcps_preddat);
8725         tp->rcv_nxt += tlen;
8726         /*
8727          * Pull snd_wl1 up to prevent seq wrap relative to th_seq.
8728          */
8729         tp->snd_wl1 = th->th_seq;
8730         /*
8731          * Pull rcv_up up to prevent seq wrap relative to rcv_nxt.
8732          */
8733         tp->rcv_up = tp->rcv_nxt;
8734         KMOD_TCPSTAT_ADD(tcps_rcvpack, (int)nsegs);
8735         KMOD_TCPSTAT_ADD(tcps_rcvbyte, tlen);
8736 #ifdef TCPDEBUG
8737         if (so->so_options & SO_DEBUG)
8738                 tcp_trace(TA_INPUT, ostate, tp,
8739                     (void *)tcp_saveipgen, &tcp_savetcp, 0);
8740 #endif
8741         newsize = tcp_autorcvbuf(m, th, so, tp, tlen);
8742
8743         /* Add data to socket buffer. */
8744         SOCKBUF_LOCK(&so->so_rcv);
8745         if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
8746                 m_freem(m);
8747         } else {
8748                 /*
8749                  * Set new socket buffer size. Give up when limit is
8750                  * reached.
8751                  */
8752                 if (newsize)
8753                         if (!sbreserve_locked(&so->so_rcv,
8754                             newsize, so, NULL))
8755                                 so->so_rcv.sb_flags &= ~SB_AUTOSIZE;
8756                 m_adj(m, drop_hdrlen);  /* delayed header drop */
8757
8758 #ifdef NETFLIX_SB_LIMITS
8759                 appended =
8760 #endif
8761                         sbappendstream_locked(&so->so_rcv, m, 0);
8762                 ctf_calc_rwin(so, tp);
8763         }
8764         /* NB: sorwakeup_locked() does an implicit unlock. */
8765         sorwakeup_locked(so);
8766 #ifdef NETFLIX_SB_LIMITS
8767         if (so->so_rcv.sb_shlim && mcnt != appended)
8768                 counter_fo_release(so->so_rcv.sb_shlim, mcnt - appended);
8769 #endif
8770         if (DELAY_ACK(tp, bbr, nsegs)) {
8771                 bbr->bbr_segs_rcvd += max(1, nsegs);
8772                 tp->t_flags |= TF_DELACK;
8773                 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8774         } else {
8775                 bbr->r_wanted_output = 1;
8776                 tp->t_flags |= TF_ACKNOW;
8777         }
8778         return (1);
8779 }
8780
8781 /*
8782  * This subfunction is used to try to highly optimize the
8783  * fast path. We again allow window updates that are
8784  * in sequence to remain in the fast-path. We also add
8785  * in the __predict's to attempt to help the compiler.
8786  * Note that if we return a 0, then we can *not* process
8787  * it and the caller should push the packet into the
8788  * slow-path. If we return 1, then all is well and
8789  * the packet is fully processed.
8790  */
8791 static int
8792 bbr_fastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
8793     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8794     uint32_t tiwin, int32_t nxt_pkt)
8795 {
8796         int32_t acked;
8797         uint16_t nsegs;
8798         uint32_t sack_changed;
8799 #ifdef TCPDEBUG
8800         /*
8801          * The size of tcp_saveipgen must be the size of the max ip header,
8802          * now IPv6.
8803          */
8804         u_char tcp_saveipgen[IP6_HDR_LEN];
8805         struct tcphdr tcp_savetcp;
8806         short ostate = 0;
8807
8808 #endif
8809         uint32_t prev_acked = 0;
8810         struct tcp_bbr *bbr;
8811
8812         if (__predict_false(SEQ_LEQ(th->th_ack, tp->snd_una))) {
8813                 /* Old ack, behind (or duplicate to) the last one rcv'd */
8814                 return (0);
8815         }
8816         if (__predict_false(SEQ_GT(th->th_ack, tp->snd_max))) {
8817                 /* Above what we have sent? */
8818                 return (0);
8819         }
8820         if (__predict_false(tiwin == 0)) {
8821                 /* zero window */
8822                 return (0);
8823         }
8824         if (__predict_false(tp->t_flags & (TF_NEEDSYN | TF_NEEDFIN))) {
8825                 /* We need a SYN or a FIN, unlikely.. */
8826                 return (0);
8827         }
8828         if ((to->to_flags & TOF_TS) && __predict_false(TSTMP_LT(to->to_tsval, tp->ts_recent))) {
8829                 /* Timestamp is behind .. old ack with seq wrap? */
8830                 return (0);
8831         }
8832         if (__predict_false(IN_RECOVERY(tp->t_flags))) {
8833                 /* Still recovering */
8834                 return (0);
8835         }
8836         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8837         if (__predict_false(bbr->r_ctl.rc_resend != NULL)) {
8838                 /* We are retransmitting */
8839                 return (0);
8840         }
8841         if (__predict_false(bbr->rc_in_persist != 0)) {
8842                 /* In persist mode */
8843                 return (0);
8844         }
8845         if (bbr->r_ctl.rc_sacked) {
8846                 /* We have sack holes on our scoreboard */
8847                 return (0);
8848         }
8849         /* Ok if we reach here, we can process a fast-ack */
8850         nsegs = max(1, m->m_pkthdr.lro_nsegs);
8851         sack_changed = bbr_log_ack(tp, to, th, &prev_acked);
8852         /*
8853          * We never detect loss in fast ack [we can't
8854          * have a sack and can't be in recovery so
8855          * we always pass 0 (nothing detected)].
8856          */
8857         bbr_lt_bw_sampling(bbr, bbr->r_ctl.rc_rcvtime, 0);
8858         /* Did the window get updated? */
8859         if (tiwin != tp->snd_wnd) {
8860                 tp->snd_wnd = tiwin;
8861                 tp->snd_wl1 = th->th_seq;
8862                 if (tp->snd_wnd > tp->max_sndwnd)
8863                         tp->max_sndwnd = tp->snd_wnd;
8864         }
8865         /* Do we need to exit persists? */
8866         if ((bbr->rc_in_persist != 0) &&
8867             (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
8868                                bbr_minseg(bbr)))) {
8869                 bbr_exit_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8870                 bbr->r_wanted_output = 1;
8871         }
8872         /* Do we need to enter persists? */
8873         if ((bbr->rc_in_persist == 0) &&
8874             (tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
8875             TCPS_HAVEESTABLISHED(tp->t_state) &&
8876             (tp->snd_max == tp->snd_una) &&
8877             sbavail(&tp->t_inpcb->inp_socket->so_snd) &&
8878             (sbavail(&tp->t_inpcb->inp_socket->so_snd) > tp->snd_wnd)) {
8879                 /* No send window.. we must enter persist */
8880                 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
8881         }
8882         /*
8883          * If last ACK falls within this segment's sequence numbers, record
8884          * the timestamp. NOTE that the test is modified according to the
8885          * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
8886          */
8887         if ((to->to_flags & TOF_TS) != 0 &&
8888             SEQ_LEQ(th->th_seq, tp->last_ack_sent)) {
8889                 tp->ts_recent_age = bbr->r_ctl.rc_rcvtime;
8890                 tp->ts_recent = to->to_tsval;
8891         }
8892         /*
8893          * This is a pure ack for outstanding data.
8894          */
8895         KMOD_TCPSTAT_INC(tcps_predack);
8896
8897         /*
8898          * "bad retransmit" recovery.
8899          */
8900         if (tp->t_flags & TF_PREVVALID) {
8901                 tp->t_flags &= ~TF_PREVVALID;
8902                 if (tp->t_rxtshift == 1 &&
8903                     (int)(ticks - tp->t_badrxtwin) < 0)
8904                         bbr_cong_signal(tp, th, CC_RTO_ERR, NULL);
8905         }
8906         /*
8907          * Recalculate the transmit timer / rtt.
8908          *
8909          * Some boxes send broken timestamp replies during the SYN+ACK
8910          * phase, ignore timestamps of 0 or we could calculate a huge RTT
8911          * and blow up the retransmit timer.
8912          */
8913         acked = BYTES_THIS_ACK(tp, th);
8914
8915 #ifdef TCP_HHOOK
8916         /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */
8917         hhook_run_tcp_est_in(tp, th, to);
8918 #endif
8919
8920         KMOD_TCPSTAT_ADD(tcps_rcvackpack, (int)nsegs);
8921         KMOD_TCPSTAT_ADD(tcps_rcvackbyte, acked);
8922         sbdrop(&so->so_snd, acked);
8923
8924         if (SEQ_GT(th->th_ack, tp->snd_una))
8925                 bbr_collapse_rtt(tp, bbr, TCP_REXMTVAL(tp));
8926         tp->snd_una = th->th_ack;
8927         if (tp->snd_wnd < ctf_outstanding(tp))
8928                 /* The peer collapsed its window on us */
8929                 bbr_collapsed_window(bbr);
8930         else if (bbr->rc_has_collapsed)
8931                 bbr_un_collapse_window(bbr);
8932
8933         if (SEQ_GT(tp->snd_una, tp->snd_recover)) {
8934                 tp->snd_recover = tp->snd_una;
8935         }
8936         bbr_ack_received(tp, bbr, th, acked, sack_changed, prev_acked, __LINE__, 0);
8937         /*
8938          * Pull snd_wl2 up to prevent seq wrap relative to th_ack.
8939          */
8940         tp->snd_wl2 = th->th_ack;
8941         m_freem(m);
8942         /*
8943          * If all outstanding data are acked, stop retransmit timer,
8944          * otherwise restart timer using current (possibly backed-off)
8945          * value. If process is waiting for space, wakeup/selwakeup/signal.
8946          * If data are ready to send, let tcp_output decide between more
8947          * output or persist.
8948          */
8949 #ifdef TCPDEBUG
8950         if (so->so_options & SO_DEBUG)
8951                 tcp_trace(TA_INPUT, ostate, tp,
8952                     (void *)tcp_saveipgen,
8953                     &tcp_savetcp, 0);
8954 #endif
8955         /* Wake up the socket if we have room to write more */
8956         sowwakeup(so);
8957         if (tp->snd_una == tp->snd_max) {
8958                 /* Nothing left outstanding */
8959                 bbr_log_progress_event(bbr, tp, ticks, PROGRESS_CLEAR, __LINE__);
8960                 if (sbavail(&tp->t_inpcb->inp_socket->so_snd) == 0)
8961                         bbr->rc_tp->t_acktime = 0;
8962                 bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
8963                 if (bbr->rc_in_persist == 0) {
8964                         bbr->r_ctl.rc_went_idle_time = bbr->r_ctl.rc_rcvtime;
8965                 }
8966                 sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
8967                 bbr_log_ack_clear(bbr, bbr->r_ctl.rc_rcvtime);
8968                 /*
8969                  * We invalidate the last ack here since we
8970                  * don't want to transfer forward the time
8971                  * for our sum's calculations.
8972                  */
8973                 bbr->r_wanted_output = 1;
8974         }
8975         if (sbavail(&so->so_snd)) {
8976                 bbr->r_wanted_output = 1;
8977         }
8978         return (1);
8979 }
8980
8981 /*
8982  * Return value of 1, the TCB is unlocked and most
8983  * likely gone, return value of 0, the TCB is still
8984  * locked.
8985  */
8986 static int
8987 bbr_do_syn_sent(struct mbuf *m, struct tcphdr *th, struct socket *so,
8988     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
8989     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
8990 {
8991         int32_t todrop;
8992         int32_t ourfinisacked = 0;
8993         struct tcp_bbr *bbr;
8994         int32_t ret_val = 0;
8995
8996         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
8997         ctf_calc_rwin(so, tp);
8998         /*
8999          * If the state is SYN_SENT: if seg contains an ACK, but not for our
9000          * SYN, drop the input. if seg contains a RST, then drop the
9001          * connection. if seg does not contain SYN, then drop it. Otherwise
9002          * this is an acceptable SYN segment initialize tp->rcv_nxt and
9003          * tp->irs if seg contains ack then advance tp->snd_una. BRR does
9004          * not support ECN so we will not say we are capable. if SYN has
9005          * been acked change to ESTABLISHED else SYN_RCVD state arrange for
9006          * segment to be acked (eventually) continue processing rest of
9007          * data/controls, beginning with URG
9008          */
9009         if ((thflags & TH_ACK) &&
9010             (SEQ_LEQ(th->th_ack, tp->iss) ||
9011             SEQ_GT(th->th_ack, tp->snd_max))) {
9012                 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9013                 return (1);
9014         }
9015         if ((thflags & (TH_ACK | TH_RST)) == (TH_ACK | TH_RST)) {
9016                 TCP_PROBE5(connect__refused, NULL, tp,
9017                     mtod(m, const char *), tp, th);
9018                 tp = tcp_drop(tp, ECONNREFUSED);
9019                 ctf_do_drop(m, tp);
9020                 return (1);
9021         }
9022         if (thflags & TH_RST) {
9023                 ctf_do_drop(m, tp);
9024                 return (1);
9025         }
9026         if (!(thflags & TH_SYN)) {
9027                 ctf_do_drop(m, tp);
9028                 return (1);
9029         }
9030         tp->irs = th->th_seq;
9031         tcp_rcvseqinit(tp);
9032         if (thflags & TH_ACK) {
9033                 int tfo_partial = 0;
9034
9035                 KMOD_TCPSTAT_INC(tcps_connects);
9036                 soisconnected(so);
9037 #ifdef MAC
9038                 mac_socketpeer_set_from_mbuf(m, so);
9039 #endif
9040                 /* Do window scaling on this connection? */
9041                 if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
9042                     (TF_RCVD_SCALE | TF_REQ_SCALE)) {
9043                         tp->rcv_scale = tp->request_r_scale;
9044                 }
9045                 tp->rcv_adv += min(tp->rcv_wnd,
9046                     TCP_MAXWIN << tp->rcv_scale);
9047                 /*
9048                  * If not all the data that was sent in the TFO SYN
9049                  * has been acked, resend the remainder right away.
9050                  */
9051                 if (IS_FASTOPEN(tp->t_flags) &&
9052                     (tp->snd_una != tp->snd_max)) {
9053                         tp->snd_nxt = th->th_ack;
9054                         tfo_partial = 1;
9055                 }
9056                 /*
9057                  * If there's data, delay ACK; if there's also a FIN ACKNOW
9058                  * will be turned on later.
9059                  */
9060                 if (DELAY_ACK(tp, bbr, 1) && tlen != 0 && (tfo_partial == 0)) {
9061                         bbr->bbr_segs_rcvd += 1;
9062                         tp->t_flags |= TF_DELACK;
9063                         bbr_timer_cancel(bbr, __LINE__, bbr->r_ctl.rc_rcvtime);
9064                 } else {
9065                         bbr->r_wanted_output = 1;
9066                         tp->t_flags |= TF_ACKNOW;
9067                 }
9068                 if (SEQ_GT(th->th_ack, tp->iss)) {
9069                         /*
9070                          * The SYN is acked
9071                          * handle it specially.
9072                          */
9073                         bbr_log_syn(tp, to);
9074                 }
9075                 if (SEQ_GT(th->th_ack, tp->snd_una)) {
9076                         /*
9077                          * We advance snd_una for the
9078                          * fast open case. If th_ack is
9079                          * acknowledging data beyond
9080                          * snd_una we can't just call
9081                          * ack-processing since the
9082                          * data stream in our send-map
9083                          * will start at snd_una + 1 (one
9084                          * beyond the SYN). If its just
9085                          * equal we don't need to do that
9086                          * and there is no send_map.
9087                          */
9088                         tp->snd_una++;
9089                 }
9090                 /*
9091                  * Received <SYN,ACK> in SYN_SENT[*] state. Transitions:
9092                  * SYN_SENT  --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1
9093                  */
9094                 tp->t_starttime = ticks;
9095                 if (tp->t_flags & TF_NEEDFIN) {
9096                         tcp_state_change(tp, TCPS_FIN_WAIT_1);
9097                         tp->t_flags &= ~TF_NEEDFIN;
9098                         thflags &= ~TH_SYN;
9099                 } else {
9100                         tcp_state_change(tp, TCPS_ESTABLISHED);
9101                         TCP_PROBE5(connect__established, NULL, tp,
9102                             mtod(m, const char *), tp, th);
9103                         cc_conn_init(tp);
9104                 }
9105         } else {
9106                 /*
9107                  * Received initial SYN in SYN-SENT[*] state => simultaneous
9108                  * open.  If segment contains CC option and there is a
9109                  * cached CC, apply TAO test. If it succeeds, connection is *
9110                  * half-synchronized. Otherwise, do 3-way handshake:
9111                  * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If
9112                  * there was no CC option, clear cached CC value.
9113                  */
9114                 tp->t_flags |= (TF_ACKNOW | TF_NEEDSYN);
9115                 tcp_state_change(tp, TCPS_SYN_RECEIVED);
9116         }
9117         INP_WLOCK_ASSERT(tp->t_inpcb);
9118         /*
9119          * Advance th->th_seq to correspond to first data byte. If data,
9120          * trim to stay within window, dropping FIN if necessary.
9121          */
9122         th->th_seq++;
9123         if (tlen > tp->rcv_wnd) {
9124                 todrop = tlen - tp->rcv_wnd;
9125                 m_adj(m, -todrop);
9126                 tlen = tp->rcv_wnd;
9127                 thflags &= ~TH_FIN;
9128                 KMOD_TCPSTAT_INC(tcps_rcvpackafterwin);
9129                 KMOD_TCPSTAT_ADD(tcps_rcvbyteafterwin, todrop);
9130         }
9131         tp->snd_wl1 = th->th_seq - 1;
9132         tp->rcv_up = th->th_seq;
9133         /*
9134          * Client side of transaction: already sent SYN and data. If the
9135          * remote host used T/TCP to validate the SYN, our data will be
9136          * ACK'd; if so, enter normal data segment processing in the middle
9137          * of step 5, ack processing. Otherwise, goto step 6.
9138          */
9139         if (thflags & TH_ACK) {
9140                 if ((to->to_flags & TOF_TS) != 0) {
9141                         uint32_t t, rtt;
9142
9143                         t = tcp_tv_to_mssectick(&bbr->rc_tv);
9144                         if (TSTMP_GEQ(t, to->to_tsecr)) {
9145                                 rtt = t - to->to_tsecr;
9146                                 if (rtt == 0) {
9147                                         rtt = 1;
9148                                 }
9149                                 rtt *= MS_IN_USEC;
9150                                 tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0);
9151                                 apply_filter_min_small(&bbr->r_ctl.rc_rttprop,
9152                                                        rtt, bbr->r_ctl.rc_rcvtime);
9153                         }
9154                 }
9155                 if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val))
9156                         return (ret_val);
9157                 /* We may have changed to FIN_WAIT_1 above */
9158                 if (tp->t_state == TCPS_FIN_WAIT_1) {
9159                         /*
9160                          * In FIN_WAIT_1 STATE in addition to the processing
9161                          * for the ESTABLISHED state if our FIN is now
9162                          * acknowledged then enter FIN_WAIT_2.
9163                          */
9164                         if (ourfinisacked) {
9165                                 /*
9166                                  * If we can't receive any more data, then
9167                                  * closing user can proceed. Starting the
9168                                  * timer is contrary to the specification,
9169                                  * but if we don't get a FIN we'll hang
9170                                  * forever.
9171                                  *
9172                                  * XXXjl: we should release the tp also, and
9173                                  * use a compressed state.
9174                                  */
9175                                 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
9176                                         soisdisconnected(so);
9177                                         tcp_timer_activate(tp, TT_2MSL,
9178                                             (tcp_fast_finwait2_recycle ?
9179                                             tcp_finwait2_timeout :
9180                                             TP_MAXIDLE(tp)));
9181                                 }
9182                                 tcp_state_change(tp, TCPS_FIN_WAIT_2);
9183                         }
9184                 }
9185         }
9186         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9187             tiwin, thflags, nxt_pkt));
9188 }
9189
9190 /*
9191  * Return value of 1, the TCB is unlocked and most
9192  * likely gone, return value of 0, the TCB is still
9193  * locked.
9194  */
9195 static int
9196 bbr_do_syn_recv(struct mbuf *m, struct tcphdr *th, struct socket *so,
9197                 struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9198                 uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
9199 {
9200         int32_t ourfinisacked = 0;
9201         int32_t ret_val;
9202         struct tcp_bbr *bbr;
9203
9204         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9205         ctf_calc_rwin(so, tp);
9206         if ((thflags & TH_ACK) &&
9207             (SEQ_LEQ(th->th_ack, tp->snd_una) ||
9208              SEQ_GT(th->th_ack, tp->snd_max))) {
9209                 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9210                 return (1);
9211         }
9212         if (IS_FASTOPEN(tp->t_flags)) {
9213                 /*
9214                  * When a TFO connection is in SYN_RECEIVED, the only valid
9215                  * packets are the initial SYN, a retransmit/copy of the
9216                  * initial SYN (possibly with a subset of the original
9217                  * data), a valid ACK, a FIN, or a RST.
9218                  */
9219                 if ((thflags & (TH_SYN | TH_ACK)) == (TH_SYN | TH_ACK)) {
9220                         ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9221                         return (1);
9222                 } else if (thflags & TH_SYN) {
9223                         /* non-initial SYN is ignored */
9224                         if ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_RXT) ||
9225                             (bbr->r_ctl.rc_hpts_flags & PACE_TMR_TLP) ||
9226                             (bbr->r_ctl.rc_hpts_flags & PACE_TMR_RACK)) {
9227                                 ctf_do_drop(m, NULL);
9228                                 return (0);
9229                         }
9230                 } else if (!(thflags & (TH_ACK | TH_FIN | TH_RST))) {
9231                         ctf_do_drop(m, NULL);
9232                         return (0);
9233                 }
9234         }
9235         if ((thflags & TH_RST) ||
9236             (tp->t_fin_is_rst && (thflags & TH_FIN)))
9237                 return (ctf_process_rst(m, th, so, tp));
9238         /*
9239          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9240          * it's less than ts_recent, drop it.
9241          */
9242         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9243             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9244                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9245                         return (ret_val);
9246         }
9247         /*
9248          * In the SYN-RECEIVED state, validate that the packet belongs to
9249          * this connection before trimming the data to fit the receive
9250          * window.  Check the sequence number versus IRS since we know the
9251          * sequence numbers haven't wrapped.  This is a partial fix for the
9252          * "LAND" DoS attack.
9253          */
9254         if (SEQ_LT(th->th_seq, tp->irs)) {
9255                 ctf_do_dropwithreset(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9256                 return (1);
9257         }
9258         INP_WLOCK_ASSERT(tp->t_inpcb);
9259         if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9260                 return (ret_val);
9261         }
9262         /*
9263          * If last ACK falls within this segment's sequence numbers, record
9264          * its timestamp. NOTE: 1) That the test incorporates suggestions
9265          * from the latest proposal of the tcplw@cray.com list (Braden
9266          * 1993/04/26). 2) That updating only on newer timestamps interferes
9267          * with our earlier PAWS tests, so this check should be solely
9268          * predicated on the sequence space of this segment. 3) That we
9269          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9270          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9271          * SEG.Len, This modified check allows us to overcome RFC1323's
9272          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9273          * p.869. In such cases, we can still calculate the RTT correctly
9274          * when RCV.NXT == Last.ACK.Sent.
9275          */
9276         if ((to->to_flags & TOF_TS) != 0 &&
9277             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9278             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9279                     ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9280                 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9281                 tp->ts_recent = to->to_tsval;
9282         }
9283         tp->snd_wnd = tiwin;
9284         /*
9285          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
9286          * is on (half-synchronized state), then queue data for later
9287          * processing; else drop segment and return.
9288          */
9289         if ((thflags & TH_ACK) == 0) {
9290                 if (IS_FASTOPEN(tp->t_flags)) {
9291                         cc_conn_init(tp);
9292                 }
9293                 return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9294                                          tiwin, thflags, nxt_pkt));
9295         }
9296         KMOD_TCPSTAT_INC(tcps_connects);
9297         soisconnected(so);
9298         /* Do window scaling? */
9299         if ((tp->t_flags & (TF_RCVD_SCALE | TF_REQ_SCALE)) ==
9300             (TF_RCVD_SCALE | TF_REQ_SCALE)) {
9301                 tp->rcv_scale = tp->request_r_scale;
9302         }
9303         /*
9304          * ok for the first time in lets see if we can use the ts to figure
9305          * out what the initial RTT was.
9306          */
9307         if ((to->to_flags & TOF_TS) != 0) {
9308                 uint32_t t, rtt;
9309
9310                 t = tcp_tv_to_mssectick(&bbr->rc_tv);
9311                 if (TSTMP_GEQ(t, to->to_tsecr)) {
9312                         rtt = t - to->to_tsecr;
9313                         if (rtt == 0) {
9314                                 rtt = 1;
9315                         }
9316                         rtt *= MS_IN_USEC;
9317                         tcp_bbr_xmit_timer(bbr, rtt, 0, 0, 0);
9318                         apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, bbr->r_ctl.rc_rcvtime);
9319                 }
9320         }
9321         /* Drop off any SYN in the send map (probably not there)  */
9322         if (thflags & TH_ACK)
9323                 bbr_log_syn(tp, to);
9324         if (IS_FASTOPEN(tp->t_flags) && tp->t_tfo_pending) {
9325
9326                 tcp_fastopen_decrement_counter(tp->t_tfo_pending);
9327                 tp->t_tfo_pending = NULL;
9328                 /*
9329                  * Account for the ACK of our SYN prior to regular
9330                  * ACK processing below.
9331                  */
9332                 tp->snd_una++;
9333         }
9334         /*
9335          * Make transitions: SYN-RECEIVED  -> ESTABLISHED SYN-RECEIVED* ->
9336          * FIN-WAIT-1
9337          */
9338         tp->t_starttime = ticks;
9339         if (tp->t_flags & TF_NEEDFIN) {
9340                 tcp_state_change(tp, TCPS_FIN_WAIT_1);
9341                 tp->t_flags &= ~TF_NEEDFIN;
9342         } else {
9343                 tcp_state_change(tp, TCPS_ESTABLISHED);
9344                 TCP_PROBE5(accept__established, NULL, tp,
9345                            mtod(m, const char *), tp, th);
9346                 /*
9347                  * TFO connections call cc_conn_init() during SYN
9348                  * processing.  Calling it again here for such connections
9349                  * is not harmless as it would undo the snd_cwnd reduction
9350                  * that occurs when a TFO SYN|ACK is retransmitted.
9351                  */
9352                 if (!IS_FASTOPEN(tp->t_flags))
9353                         cc_conn_init(tp);
9354         }
9355         /*
9356          * If segment contains data or ACK, will call tcp_reass() later; if
9357          * not, do so now to pass queued data to user.
9358          */
9359         if (tlen == 0 && (thflags & TH_FIN) == 0)
9360                 (void)tcp_reass(tp, (struct tcphdr *)0, NULL, 0,
9361                         (struct mbuf *)0);
9362         tp->snd_wl1 = th->th_seq - 1;
9363         if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9364                 return (ret_val);
9365         }
9366         if (tp->t_state == TCPS_FIN_WAIT_1) {
9367                 /* We could have went to FIN_WAIT_1 (or EST) above */
9368                 /*
9369                  * In FIN_WAIT_1 STATE in addition to the processing for the
9370                  * ESTABLISHED state if our FIN is now acknowledged then
9371                  * enter FIN_WAIT_2.
9372                  */
9373                 if (ourfinisacked) {
9374                         /*
9375                          * If we can't receive any more data, then closing
9376                          * user can proceed. Starting the timer is contrary
9377                          * to the specification, but if we don't get a FIN
9378                          * we'll hang forever.
9379                          *
9380                          * XXXjl: we should release the tp also, and use a
9381                          * compressed state.
9382                          */
9383                         if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
9384                                 soisdisconnected(so);
9385                                 tcp_timer_activate(tp, TT_2MSL,
9386                                                    (tcp_fast_finwait2_recycle ?
9387                                                     tcp_finwait2_timeout :
9388                                                     TP_MAXIDLE(tp)));
9389                         }
9390                         tcp_state_change(tp, TCPS_FIN_WAIT_2);
9391                 }
9392         }
9393         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9394                                  tiwin, thflags, nxt_pkt));
9395 }
9396
9397 /*
9398  * Return value of 1, the TCB is unlocked and most
9399  * likely gone, return value of 0, the TCB is still
9400  * locked.
9401  */
9402 static int
9403 bbr_do_established(struct mbuf *m, struct tcphdr *th, struct socket *so,
9404     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9405     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
9406 {
9407         struct tcp_bbr *bbr;
9408         int32_t ret_val;
9409
9410         /*
9411          * Header prediction: check for the two common cases of a
9412          * uni-directional data xfer.  If the packet has no control flags,
9413          * is in-sequence, the window didn't change and we're not
9414          * retransmitting, it's a candidate.  If the length is zero and the
9415          * ack moved forward, we're the sender side of the xfer.  Just free
9416          * the data acked & wake any higher level process that was blocked
9417          * waiting for space.  If the length is non-zero and the ack didn't
9418          * move, we're the receiver side.  If we're getting packets in-order
9419          * (the reassembly queue is empty), add the data toc The socket
9420          * buffer and note that we need a delayed ack. Make sure that the
9421          * hidden state-flags are also off. Since we check for
9422          * TCPS_ESTABLISHED first, it can only be TH_NEEDSYN.
9423          */
9424         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9425         if (bbr->r_ctl.rc_delivered < (4 * tp->t_maxseg)) {
9426                 /*
9427                  * If we have delived under 4 segments increase the initial
9428                  * window if raised by the peer. We use this to determine
9429                  * dynamic and static rwnd's at the end of a connection.
9430                  */
9431                 bbr->r_ctl.rc_init_rwnd = max(tiwin, tp->snd_wnd);
9432         }
9433         if (__predict_true(((to->to_flags & TOF_SACK) == 0)) &&
9434             __predict_true((thflags & (TH_SYN | TH_FIN | TH_RST | TH_URG | TH_ACK)) == TH_ACK) &&
9435             __predict_true(SEGQ_EMPTY(tp)) &&
9436             __predict_true(th->th_seq == tp->rcv_nxt)) {
9437                 if (tlen == 0) {
9438                         if (bbr_fastack(m, th, so, tp, to, drop_hdrlen, tlen,
9439                             tiwin, nxt_pkt)) {
9440                                 return (0);
9441                         }
9442                 } else {
9443                         if (bbr_do_fastnewdata(m, th, so, tp, to, drop_hdrlen, tlen,
9444                             tiwin, nxt_pkt)) {
9445                                 return (0);
9446                         }
9447                 }
9448         }
9449         ctf_calc_rwin(so, tp);
9450
9451         if ((thflags & TH_RST) ||
9452             (tp->t_fin_is_rst && (thflags & TH_FIN)))
9453                 return (ctf_process_rst(m, th, so, tp));
9454         /*
9455          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9456          * synchronized state.
9457          */
9458         if (thflags & TH_SYN) {
9459                 ctf_challenge_ack(m, th, tp, &ret_val);
9460                 return (ret_val);
9461         }
9462         /*
9463          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9464          * it's less than ts_recent, drop it.
9465          */
9466         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9467             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9468                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9469                         return (ret_val);
9470         }
9471         INP_WLOCK_ASSERT(tp->t_inpcb);
9472         if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9473                 return (ret_val);
9474         }
9475         /*
9476          * If last ACK falls within this segment's sequence numbers, record
9477          * its timestamp. NOTE: 1) That the test incorporates suggestions
9478          * from the latest proposal of the tcplw@cray.com list (Braden
9479          * 1993/04/26). 2) That updating only on newer timestamps interferes
9480          * with our earlier PAWS tests, so this check should be solely
9481          * predicated on the sequence space of this segment. 3) That we
9482          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9483          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9484          * SEG.Len, This modified check allows us to overcome RFC1323's
9485          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9486          * p.869. In such cases, we can still calculate the RTT correctly
9487          * when RCV.NXT == Last.ACK.Sent.
9488          */
9489         if ((to->to_flags & TOF_TS) != 0 &&
9490             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9491             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9492             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9493                 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9494                 tp->ts_recent = to->to_tsval;
9495         }
9496         /*
9497          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
9498          * is on (half-synchronized state), then queue data for later
9499          * processing; else drop segment and return.
9500          */
9501         if ((thflags & TH_ACK) == 0) {
9502                 if (tp->t_flags & TF_NEEDSYN) {
9503                         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9504                             tiwin, thflags, nxt_pkt));
9505                 } else if (tp->t_flags & TF_ACKNOW) {
9506                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9507                         bbr->r_wanted_output = 1;
9508                         return (ret_val);
9509                 } else {
9510                         ctf_do_drop(m, NULL);
9511                         return (0);
9512                 }
9513         }
9514         /*
9515          * Ack processing.
9516          */
9517         if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
9518                 return (ret_val);
9519         }
9520         if (sbavail(&so->so_snd)) {
9521                 if (bbr_progress_timeout_check(bbr)) {
9522                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9523                         return (1);
9524                 }
9525         }
9526         /* State changes only happen in bbr_process_data() */
9527         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9528             tiwin, thflags, nxt_pkt));
9529 }
9530
9531 /*
9532  * Return value of 1, the TCB is unlocked and most
9533  * likely gone, return value of 0, the TCB is still
9534  * locked.
9535  */
9536 static int
9537 bbr_do_close_wait(struct mbuf *m, struct tcphdr *th, struct socket *so,
9538     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9539     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
9540 {
9541         struct tcp_bbr *bbr;
9542         int32_t ret_val;
9543
9544         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9545         ctf_calc_rwin(so, tp);
9546         if ((thflags & TH_RST) ||
9547             (tp->t_fin_is_rst && (thflags & TH_FIN)))
9548                 return (ctf_process_rst(m, th, so, tp));
9549         /*
9550          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9551          * synchronized state.
9552          */
9553         if (thflags & TH_SYN) {
9554                 ctf_challenge_ack(m, th, tp, &ret_val);
9555                 return (ret_val);
9556         }
9557         /*
9558          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9559          * it's less than ts_recent, drop it.
9560          */
9561         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9562             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9563                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9564                         return (ret_val);
9565         }
9566         INP_WLOCK_ASSERT(tp->t_inpcb);
9567         if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9568                 return (ret_val);
9569         }
9570         /*
9571          * If last ACK falls within this segment's sequence numbers, record
9572          * its timestamp. NOTE: 1) That the test incorporates suggestions
9573          * from the latest proposal of the tcplw@cray.com list (Braden
9574          * 1993/04/26). 2) That updating only on newer timestamps interferes
9575          * with our earlier PAWS tests, so this check should be solely
9576          * predicated on the sequence space of this segment. 3) That we
9577          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9578          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9579          * SEG.Len, This modified check allows us to overcome RFC1323's
9580          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9581          * p.869. In such cases, we can still calculate the RTT correctly
9582          * when RCV.NXT == Last.ACK.Sent.
9583          */
9584         if ((to->to_flags & TOF_TS) != 0 &&
9585             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9586             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9587             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9588                 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9589                 tp->ts_recent = to->to_tsval;
9590         }
9591         /*
9592          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
9593          * is on (half-synchronized state), then queue data for later
9594          * processing; else drop segment and return.
9595          */
9596         if ((thflags & TH_ACK) == 0) {
9597                 if (tp->t_flags & TF_NEEDSYN) {
9598                         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9599                             tiwin, thflags, nxt_pkt));
9600                 } else if (tp->t_flags & TF_ACKNOW) {
9601                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9602                         bbr->r_wanted_output = 1;
9603                         return (ret_val);
9604                 } else {
9605                         ctf_do_drop(m, NULL);
9606                         return (0);
9607                 }
9608         }
9609         /*
9610          * Ack processing.
9611          */
9612         if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, NULL, thflags, &ret_val)) {
9613                 return (ret_val);
9614         }
9615         if (sbavail(&so->so_snd)) {
9616                 if (bbr_progress_timeout_check(bbr)) {
9617                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9618                         return (1);
9619                 }
9620         }
9621         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9622             tiwin, thflags, nxt_pkt));
9623 }
9624
9625 static int
9626 bbr_check_data_after_close(struct mbuf *m, struct tcp_bbr *bbr,
9627     struct tcpcb *tp, int32_t * tlen, struct tcphdr *th, struct socket *so)
9628 {
9629
9630         if (bbr->rc_allow_data_af_clo == 0) {
9631 close_now:
9632                 tp = tcp_close(tp);
9633                 KMOD_TCPSTAT_INC(tcps_rcvafterclose);
9634                 ctf_do_dropwithreset(m, tp, th, BANDLIM_UNLIMITED, (*tlen));
9635                 return (1);
9636         }
9637         if (sbavail(&so->so_snd) == 0)
9638                 goto close_now;
9639         /* Ok we allow data that is ignored and a followup reset */
9640         tp->rcv_nxt = th->th_seq + *tlen;
9641         tp->t_flags2 |= TF2_DROP_AF_DATA;
9642         bbr->r_wanted_output = 1;
9643         *tlen = 0;
9644         return (0);
9645 }
9646
9647 /*
9648  * Return value of 1, the TCB is unlocked and most
9649  * likely gone, return value of 0, the TCB is still
9650  * locked.
9651  */
9652 static int
9653 bbr_do_fin_wait_1(struct mbuf *m, struct tcphdr *th, struct socket *so,
9654     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9655     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
9656 {
9657         int32_t ourfinisacked = 0;
9658         int32_t ret_val;
9659         struct tcp_bbr *bbr;
9660
9661         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9662         ctf_calc_rwin(so, tp);
9663         if ((thflags & TH_RST) ||
9664             (tp->t_fin_is_rst && (thflags & TH_FIN)))
9665                 return (ctf_process_rst(m, th, so, tp));
9666         /*
9667          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9668          * synchronized state.
9669          */
9670         if (thflags & TH_SYN) {
9671                 ctf_challenge_ack(m, th, tp, &ret_val);
9672                 return (ret_val);
9673         }
9674         /*
9675          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9676          * it's less than ts_recent, drop it.
9677          */
9678         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9679             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9680                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9681                         return (ret_val);
9682         }
9683         INP_WLOCK_ASSERT(tp->t_inpcb);
9684         if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9685                 return (ret_val);
9686         }
9687         /*
9688          * If new data are received on a connection after the user processes
9689          * are gone, then RST the other end.
9690          */
9691         if ((so->so_state & SS_NOFDREF) && tlen) {
9692                 /*
9693                  * We call a new function now so we might continue and setup
9694                  * to reset at all data being ack'd.
9695                  */
9696                 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9697                         return (1);
9698         }
9699         /*
9700          * If last ACK falls within this segment's sequence numbers, record
9701          * its timestamp. NOTE: 1) That the test incorporates suggestions
9702          * from the latest proposal of the tcplw@cray.com list (Braden
9703          * 1993/04/26). 2) That updating only on newer timestamps interferes
9704          * with our earlier PAWS tests, so this check should be solely
9705          * predicated on the sequence space of this segment. 3) That we
9706          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9707          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9708          * SEG.Len, This modified check allows us to overcome RFC1323's
9709          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9710          * p.869. In such cases, we can still calculate the RTT correctly
9711          * when RCV.NXT == Last.ACK.Sent.
9712          */
9713         if ((to->to_flags & TOF_TS) != 0 &&
9714             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9715             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9716             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9717                 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9718                 tp->ts_recent = to->to_tsval;
9719         }
9720         /*
9721          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
9722          * is on (half-synchronized state), then queue data for later
9723          * processing; else drop segment and return.
9724          */
9725         if ((thflags & TH_ACK) == 0) {
9726                 if (tp->t_flags & TF_NEEDSYN) {
9727                         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9728                             tiwin, thflags, nxt_pkt));
9729                 } else if (tp->t_flags & TF_ACKNOW) {
9730                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9731                         bbr->r_wanted_output = 1;
9732                         return (ret_val);
9733                 } else {
9734                         ctf_do_drop(m, NULL);
9735                         return (0);
9736                 }
9737         }
9738         /*
9739          * Ack processing.
9740          */
9741         if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9742                 return (ret_val);
9743         }
9744         if (ourfinisacked) {
9745                 /*
9746                  * If we can't receive any more data, then closing user can
9747                  * proceed. Starting the timer is contrary to the
9748                  * specification, but if we don't get a FIN we'll hang
9749                  * forever.
9750                  *
9751                  * XXXjl: we should release the tp also, and use a
9752                  * compressed state.
9753                  */
9754                 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
9755                         soisdisconnected(so);
9756                         tcp_timer_activate(tp, TT_2MSL,
9757                             (tcp_fast_finwait2_recycle ?
9758                             tcp_finwait2_timeout :
9759                             TP_MAXIDLE(tp)));
9760                 }
9761                 tcp_state_change(tp, TCPS_FIN_WAIT_2);
9762         }
9763         if (sbavail(&so->so_snd)) {
9764                 if (bbr_progress_timeout_check(bbr)) {
9765                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9766                         return (1);
9767                 }
9768         }
9769         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9770             tiwin, thflags, nxt_pkt));
9771 }
9772
9773 /*
9774  * Return value of 1, the TCB is unlocked and most
9775  * likely gone, return value of 0, the TCB is still
9776  * locked.
9777  */
9778 static int
9779 bbr_do_closing(struct mbuf *m, struct tcphdr *th, struct socket *so,
9780     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9781     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
9782 {
9783         int32_t ourfinisacked = 0;
9784         int32_t ret_val;
9785         struct tcp_bbr *bbr;
9786
9787         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9788         ctf_calc_rwin(so, tp);
9789         if ((thflags & TH_RST) ||
9790             (tp->t_fin_is_rst && (thflags & TH_FIN)))
9791                 return (ctf_process_rst(m, th, so, tp));
9792         /*
9793          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9794          * synchronized state.
9795          */
9796         if (thflags & TH_SYN) {
9797                 ctf_challenge_ack(m, th, tp, &ret_val);
9798                 return (ret_val);
9799         }
9800         /*
9801          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9802          * it's less than ts_recent, drop it.
9803          */
9804         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9805             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9806                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9807                         return (ret_val);
9808         }
9809         INP_WLOCK_ASSERT(tp->t_inpcb);
9810         if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9811                 return (ret_val);
9812         }
9813         /*
9814          * If new data are received on a connection after the user processes
9815          * are gone, then RST the other end.
9816          */
9817         if ((so->so_state & SS_NOFDREF) && tlen) {
9818                 /*
9819                  * We call a new function now so we might continue and setup
9820                  * to reset at all data being ack'd.
9821                  */
9822                 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9823                         return (1);
9824         }
9825         /*
9826          * If last ACK falls within this segment's sequence numbers, record
9827          * its timestamp. NOTE: 1) That the test incorporates suggestions
9828          * from the latest proposal of the tcplw@cray.com list (Braden
9829          * 1993/04/26). 2) That updating only on newer timestamps interferes
9830          * with our earlier PAWS tests, so this check should be solely
9831          * predicated on the sequence space of this segment. 3) That we
9832          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9833          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9834          * SEG.Len, This modified check allows us to overcome RFC1323's
9835          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9836          * p.869. In such cases, we can still calculate the RTT correctly
9837          * when RCV.NXT == Last.ACK.Sent.
9838          */
9839         if ((to->to_flags & TOF_TS) != 0 &&
9840             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9841             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9842             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9843                 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9844                 tp->ts_recent = to->to_tsval;
9845         }
9846         /*
9847          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
9848          * is on (half-synchronized state), then queue data for later
9849          * processing; else drop segment and return.
9850          */
9851         if ((thflags & TH_ACK) == 0) {
9852                 if (tp->t_flags & TF_NEEDSYN) {
9853                         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9854                             tiwin, thflags, nxt_pkt));
9855                 } else if (tp->t_flags & TF_ACKNOW) {
9856                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9857                         bbr->r_wanted_output = 1;
9858                         return (ret_val);
9859                 } else {
9860                         ctf_do_drop(m, NULL);
9861                         return (0);
9862                 }
9863         }
9864         /*
9865          * Ack processing.
9866          */
9867         if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9868                 return (ret_val);
9869         }
9870         if (ourfinisacked) {
9871                 tcp_twstart(tp);
9872                 m_freem(m);
9873                 return (1);
9874         }
9875         if (sbavail(&so->so_snd)) {
9876                 if (bbr_progress_timeout_check(bbr)) {
9877                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9878                         return (1);
9879                 }
9880         }
9881         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9882             tiwin, thflags, nxt_pkt));
9883 }
9884
9885 /*
9886  * Return value of 1, the TCB is unlocked and most
9887  * likely gone, return value of 0, the TCB is still
9888  * locked.
9889  */
9890 static int
9891 bbr_do_lastack(struct mbuf *m, struct tcphdr *th, struct socket *so,
9892     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
9893     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
9894 {
9895         int32_t ourfinisacked = 0;
9896         int32_t ret_val;
9897         struct tcp_bbr *bbr;
9898
9899         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
9900         ctf_calc_rwin(so, tp);
9901         if ((thflags & TH_RST) ||
9902             (tp->t_fin_is_rst && (thflags & TH_FIN)))
9903                 return (ctf_process_rst(m, th, so, tp));
9904         /*
9905          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
9906          * synchronized state.
9907          */
9908         if (thflags & TH_SYN) {
9909                 ctf_challenge_ack(m, th, tp, &ret_val);
9910                 return (ret_val);
9911         }
9912         /*
9913          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
9914          * it's less than ts_recent, drop it.
9915          */
9916         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
9917             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
9918                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
9919                         return (ret_val);
9920         }
9921         INP_WLOCK_ASSERT(tp->t_inpcb);
9922         if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
9923                 return (ret_val);
9924         }
9925         /*
9926          * If new data are received on a connection after the user processes
9927          * are gone, then RST the other end.
9928          */
9929         if ((so->so_state & SS_NOFDREF) && tlen) {
9930                 /*
9931                  * We call a new function now so we might continue and setup
9932                  * to reset at all data being ack'd.
9933                  */
9934                 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
9935                         return (1);
9936         }
9937         /*
9938          * If last ACK falls within this segment's sequence numbers, record
9939          * its timestamp. NOTE: 1) That the test incorporates suggestions
9940          * from the latest proposal of the tcplw@cray.com list (Braden
9941          * 1993/04/26). 2) That updating only on newer timestamps interferes
9942          * with our earlier PAWS tests, so this check should be solely
9943          * predicated on the sequence space of this segment. 3) That we
9944          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
9945          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
9946          * SEG.Len, This modified check allows us to overcome RFC1323's
9947          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
9948          * p.869. In such cases, we can still calculate the RTT correctly
9949          * when RCV.NXT == Last.ACK.Sent.
9950          */
9951         if ((to->to_flags & TOF_TS) != 0 &&
9952             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
9953             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
9954             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
9955                 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
9956                 tp->ts_recent = to->to_tsval;
9957         }
9958         /*
9959          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
9960          * is on (half-synchronized state), then queue data for later
9961          * processing; else drop segment and return.
9962          */
9963         if ((thflags & TH_ACK) == 0) {
9964                 if (tp->t_flags & TF_NEEDSYN) {
9965                         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9966                             tiwin, thflags, nxt_pkt));
9967                 } else if (tp->t_flags & TF_ACKNOW) {
9968                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
9969                         bbr->r_wanted_output = 1;
9970                         return (ret_val);
9971                 } else {
9972                         ctf_do_drop(m, NULL);
9973                         return (0);
9974                 }
9975         }
9976         /*
9977          * case TCPS_LAST_ACK: Ack processing.
9978          */
9979         if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
9980                 return (ret_val);
9981         }
9982         if (ourfinisacked) {
9983                 tp = tcp_close(tp);
9984                 ctf_do_drop(m, tp);
9985                 return (1);
9986         }
9987         if (sbavail(&so->so_snd)) {
9988                 if (bbr_progress_timeout_check(bbr)) {
9989                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
9990                         return (1);
9991                 }
9992         }
9993         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
9994             tiwin, thflags, nxt_pkt));
9995 }
9996
9997
9998 /*
9999  * Return value of 1, the TCB is unlocked and most
10000  * likely gone, return value of 0, the TCB is still
10001  * locked.
10002  */
10003 static int
10004 bbr_do_fin_wait_2(struct mbuf *m, struct tcphdr *th, struct socket *so,
10005     struct tcpcb *tp, struct tcpopt *to, int32_t drop_hdrlen, int32_t tlen,
10006     uint32_t tiwin, int32_t thflags, int32_t nxt_pkt)
10007 {
10008         int32_t ourfinisacked = 0;
10009         int32_t ret_val;
10010         struct tcp_bbr *bbr;
10011
10012         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
10013         ctf_calc_rwin(so, tp);
10014         /* Reset receive buffer auto scaling when not in bulk receive mode. */
10015         if ((thflags & TH_RST) ||
10016             (tp->t_fin_is_rst && (thflags & TH_FIN)))
10017                 return (ctf_process_rst(m, th, so, tp));
10018
10019         /*
10020          * RFC5961 Section 4.2 Send challenge ACK for any SYN in
10021          * synchronized state.
10022          */
10023         if (thflags & TH_SYN) {
10024                 ctf_challenge_ack(m, th, tp, &ret_val);
10025                 return (ret_val);
10026         }
10027         INP_WLOCK_ASSERT(tp->t_inpcb);
10028         /*
10029          * RFC 1323 PAWS: If we have a timestamp reply on this segment and
10030          * it's less than ts_recent, drop it.
10031          */
10032         if ((to->to_flags & TOF_TS) != 0 && tp->ts_recent &&
10033             TSTMP_LT(to->to_tsval, tp->ts_recent)) {
10034                 if (ctf_ts_check(m, th, tp, tlen, thflags, &ret_val))
10035                         return (ret_val);
10036         }
10037         INP_WLOCK_ASSERT(tp->t_inpcb);
10038         if (ctf_drop_checks(to, m, th, tp, &tlen, &thflags, &drop_hdrlen, &ret_val)) {
10039                 return (ret_val);
10040         }
10041         /*
10042          * If new data are received on a connection after the user processes
10043          * are gone, then we may RST the other end depending on the outcome
10044          * of bbr_check_data_after_close.
10045          */
10046         if ((so->so_state & SS_NOFDREF) &&
10047             tlen) {
10048                 /*
10049                  * We call a new function now so we might continue and setup
10050                  * to reset at all data being ack'd.
10051                  */
10052                 if (bbr_check_data_after_close(m, bbr, tp, &tlen, th, so))
10053                         return (1);
10054         }
10055         INP_WLOCK_ASSERT(tp->t_inpcb);
10056         /*
10057          * If last ACK falls within this segment's sequence numbers, record
10058          * its timestamp. NOTE: 1) That the test incorporates suggestions
10059          * from the latest proposal of the tcplw@cray.com list (Braden
10060          * 1993/04/26). 2) That updating only on newer timestamps interferes
10061          * with our earlier PAWS tests, so this check should be solely
10062          * predicated on the sequence space of this segment. 3) That we
10063          * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
10064          * + SEG.Len  instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
10065          * SEG.Len, This modified check allows us to overcome RFC1323's
10066          * limitations as described in Stevens TCP/IP Illustrated Vol. 2
10067          * p.869. In such cases, we can still calculate the RTT correctly
10068          * when RCV.NXT == Last.ACK.Sent.
10069          */
10070         INP_WLOCK_ASSERT(tp->t_inpcb);
10071         if ((to->to_flags & TOF_TS) != 0 &&
10072             SEQ_LEQ(th->th_seq, tp->last_ack_sent) &&
10073             SEQ_LEQ(tp->last_ack_sent, th->th_seq + tlen +
10074             ((thflags & (TH_SYN | TH_FIN)) != 0))) {
10075                 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
10076                 tp->ts_recent = to->to_tsval;
10077         }
10078         /*
10079          * If the ACK bit is off:  if in SYN-RECEIVED state or SENDSYN flag
10080          * is on (half-synchronized state), then queue data for later
10081          * processing; else drop segment and return.
10082          */
10083         if ((thflags & TH_ACK) == 0) {
10084                 if (tp->t_flags & TF_NEEDSYN) {
10085                         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
10086                             tiwin, thflags, nxt_pkt));
10087                 } else if (tp->t_flags & TF_ACKNOW) {
10088                         ctf_do_dropafterack(m, tp, th, thflags, tlen, &ret_val);
10089                         bbr->r_wanted_output = 1;
10090                         return (ret_val);
10091                 } else {
10092                         ctf_do_drop(m, NULL);
10093                         return (0);
10094                 }
10095         }
10096         /*
10097          * Ack processing.
10098          */
10099         INP_WLOCK_ASSERT(tp->t_inpcb);
10100         if (bbr_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val)) {
10101                 return (ret_val);
10102         }
10103         if (sbavail(&so->so_snd)) {
10104                 if (bbr_progress_timeout_check(bbr)) {
10105                         ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
10106                         return (1);
10107                 }
10108         }
10109         INP_WLOCK_ASSERT(tp->t_inpcb);
10110         return (bbr_process_data(m, th, so, tp, drop_hdrlen, tlen,
10111             tiwin, thflags, nxt_pkt));
10112 }
10113
10114 static void
10115 bbr_stop_all_timers(struct tcpcb *tp)
10116 {
10117         struct tcp_bbr *bbr;
10118
10119         /*
10120          * Assure no timers are running.
10121          */
10122         if (tcp_timer_active(tp, TT_PERSIST)) {
10123                 /* We enter in persists, set the flag appropriately */
10124                 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
10125                 bbr->rc_in_persist = 1;
10126         }
10127         tcp_timer_suspend(tp, TT_PERSIST);
10128         tcp_timer_suspend(tp, TT_REXMT);
10129         tcp_timer_suspend(tp, TT_KEEP);
10130         tcp_timer_suspend(tp, TT_DELACK);
10131 }
10132
10133 static void
10134 bbr_google_mode_on(struct tcp_bbr *bbr)
10135 {
10136         bbr->rc_use_google = 1;
10137         bbr->rc_no_pacing = 0;
10138         bbr->r_ctl.bbr_google_discount = bbr_google_discount;
10139         bbr->r_use_policer = bbr_policer_detection_enabled;
10140         bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
10141         bbr->bbr_use_rack_cheat = 0;
10142         bbr->r_ctl.rc_incr_tmrs = 0;
10143         bbr->r_ctl.rc_inc_tcp_oh = 0;
10144         bbr->r_ctl.rc_inc_ip_oh = 0;
10145         bbr->r_ctl.rc_inc_enet_oh = 0;
10146         reset_time(&bbr->r_ctl.rc_delrate,
10147                    BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT);
10148         reset_time_small(&bbr->r_ctl.rc_rttprop,
10149                          (11 * USECS_IN_SECOND));
10150         tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
10151 }
10152
10153 static void
10154 bbr_google_mode_off(struct tcp_bbr *bbr)
10155 {
10156         bbr->rc_use_google = 0;
10157         bbr->r_ctl.bbr_google_discount = 0;
10158         bbr->no_pacing_until = bbr_no_pacing_until;
10159         bbr->r_use_policer = 0;
10160         if (bbr->no_pacing_until)
10161                 bbr->rc_no_pacing = 1;
10162         else
10163                 bbr->rc_no_pacing = 0;
10164         if (bbr_use_rack_resend_cheat)
10165                 bbr->bbr_use_rack_cheat = 1;
10166         else
10167                 bbr->bbr_use_rack_cheat = 0;
10168         if (bbr_incr_timers)
10169                 bbr->r_ctl.rc_incr_tmrs = 1;
10170         else
10171                 bbr->r_ctl.rc_incr_tmrs = 0;
10172         if (bbr_include_tcp_oh)
10173                 bbr->r_ctl.rc_inc_tcp_oh = 1;
10174         else
10175                 bbr->r_ctl.rc_inc_tcp_oh = 0;
10176         if (bbr_include_ip_oh)
10177                 bbr->r_ctl.rc_inc_ip_oh = 1;
10178         else
10179                 bbr->r_ctl.rc_inc_ip_oh = 0;
10180         if (bbr_include_enet_oh)
10181                 bbr->r_ctl.rc_inc_enet_oh = 1;
10182         else
10183                 bbr->r_ctl.rc_inc_enet_oh = 0;
10184         bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10185         reset_time(&bbr->r_ctl.rc_delrate,
10186                    bbr_num_pktepo_for_del_limit);
10187         reset_time_small(&bbr->r_ctl.rc_rttprop,
10188                          (bbr_filter_len_sec * USECS_IN_SECOND));
10189         tcp_bbr_tso_size_check(bbr, tcp_get_usecs(&bbr->rc_tv));
10190 }
10191 /*
10192  * Return 0 on success, non-zero on failure
10193  * which indicates the error (usually no memory).
10194  */
10195 static int
10196 bbr_init(struct tcpcb *tp)
10197 {
10198         struct tcp_bbr *bbr = NULL;
10199         struct inpcb *inp;
10200         uint32_t cts;
10201
10202         tp->t_fb_ptr = uma_zalloc(bbr_pcb_zone, (M_NOWAIT | M_ZERO));
10203         if (tp->t_fb_ptr == NULL) {
10204                 /*
10205                  * We need to allocate memory but cant. The INP and INP_INFO
10206                  * locks and they are recusive (happens during setup. So a
10207                  * scheme to drop the locks fails :(
10208                  *
10209                  */
10210                 return (ENOMEM);
10211         }
10212         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
10213         bbr->rtt_valid = 0;
10214         inp = tp->t_inpcb;
10215         inp->inp_flags2 |= INP_CANNOT_DO_ECN;
10216         inp->inp_flags2 |= INP_SUPPORTS_MBUFQ;
10217         TAILQ_INIT(&bbr->r_ctl.rc_map);
10218         TAILQ_INIT(&bbr->r_ctl.rc_free);
10219         TAILQ_INIT(&bbr->r_ctl.rc_tmap);
10220         bbr->rc_tp = tp;
10221         if (tp->t_inpcb) {
10222                 bbr->rc_inp = tp->t_inpcb;
10223         }
10224         cts = tcp_get_usecs(&bbr->rc_tv);
10225         tp->t_acktime = 0;
10226         bbr->rc_allow_data_af_clo = bbr_ignore_data_after_close;
10227         bbr->r_ctl.rc_reorder_fade = bbr_reorder_fade;
10228         bbr->rc_tlp_threshold = bbr_tlp_thresh;
10229         bbr->r_ctl.rc_reorder_shift = bbr_reorder_thresh;
10230         bbr->r_ctl.rc_pkt_delay = bbr_pkt_delay;
10231         bbr->r_ctl.rc_min_to = bbr_min_to;
10232         bbr->rc_bbr_state = BBR_STATE_STARTUP;
10233         bbr->r_ctl.bbr_lost_at_state = 0;
10234         bbr->r_ctl.rc_lost_at_startup = 0;
10235         bbr->rc_all_timers_stopped = 0;
10236         bbr->r_ctl.rc_bbr_lastbtlbw = 0;
10237         bbr->r_ctl.rc_pkt_epoch_del = 0;
10238         bbr->r_ctl.rc_pkt_epoch = 0;
10239         bbr->r_ctl.rc_lowest_rtt = 0xffffffff;
10240         bbr->r_ctl.rc_bbr_hptsi_gain = bbr_high_gain;
10241         bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
10242         bbr->r_ctl.rc_went_idle_time = cts;
10243         bbr->rc_pacer_started = cts;
10244         bbr->r_ctl.rc_pkt_epoch_time = cts;
10245         bbr->r_ctl.rc_rcvtime = cts;
10246         bbr->r_ctl.rc_bbr_state_time = cts;
10247         bbr->r_ctl.rc_del_time = cts;
10248         bbr->r_ctl.rc_tlp_rxt_last_time = cts;
10249         bbr->r_ctl.last_in_probertt = cts;
10250         bbr->skip_gain = 0;
10251         bbr->gain_is_limited = 0;
10252         bbr->no_pacing_until = bbr_no_pacing_until;
10253         if (bbr->no_pacing_until)
10254                 bbr->rc_no_pacing = 1;
10255         if (bbr_use_google_algo) {
10256                 bbr->rc_no_pacing = 0;
10257                 bbr->rc_use_google = 1;
10258                 bbr->r_ctl.bbr_google_discount = bbr_google_discount;
10259                 bbr->r_use_policer = bbr_policer_detection_enabled;
10260         } else {
10261                 bbr->rc_use_google = 0;
10262                 bbr->r_ctl.bbr_google_discount = 0;
10263                 bbr->r_use_policer = 0;
10264         }
10265         if (bbr_ts_limiting)
10266                 bbr->rc_use_ts_limit = 1;
10267         else
10268                 bbr->rc_use_ts_limit = 0;
10269         if (bbr_ts_can_raise)
10270                 bbr->ts_can_raise = 1;
10271         else
10272                 bbr->ts_can_raise = 0;
10273         if (V_tcp_delack_enabled == 1)
10274                 tp->t_delayed_ack = 2;
10275         else if (V_tcp_delack_enabled == 0)
10276                 tp->t_delayed_ack = 0;
10277         else if (V_tcp_delack_enabled < 100)
10278                 tp->t_delayed_ack = V_tcp_delack_enabled;
10279         else
10280                 tp->t_delayed_ack = 2;
10281         if (bbr->rc_use_google == 0)
10282                 bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
10283         else
10284                 bbr->r_ctl.rc_probertt_int = (USECS_IN_SECOND * 10);
10285         bbr->r_ctl.rc_min_rto_ms = bbr_rto_min_ms;
10286         bbr->rc_max_rto_sec = bbr_rto_max_sec;
10287         bbr->rc_init_win = bbr_def_init_win;
10288         if (tp->t_flags & TF_REQ_TSTMP)
10289                 bbr->rc_last_options = TCP_TS_OVERHEAD;
10290         bbr->r_ctl.rc_pace_max_segs = tp->t_maxseg - bbr->rc_last_options;
10291         bbr->r_ctl.rc_high_rwnd = tp->snd_wnd;
10292         bbr->r_init_rtt = 1;
10293
10294         counter_u64_add(bbr_flows_nohdwr_pacing, 1);
10295         if (bbr_allow_hdwr_pacing)
10296                 bbr->bbr_hdw_pace_ena = 1;
10297         else
10298                 bbr->bbr_hdw_pace_ena = 0;
10299         if (bbr_sends_full_iwnd)
10300                 bbr->bbr_init_win_cheat = 1;
10301         else
10302                 bbr->bbr_init_win_cheat = 0;
10303         bbr->r_ctl.bbr_utter_max = bbr_hptsi_utter_max;
10304         bbr->r_ctl.rc_drain_pg = bbr_drain_gain;
10305         bbr->r_ctl.rc_startup_pg = bbr_high_gain;
10306         bbr->rc_loss_exit = bbr_exit_startup_at_loss;
10307         bbr->r_ctl.bbr_rttprobe_gain_val = bbr_rttprobe_gain;
10308         bbr->r_ctl.bbr_hptsi_per_second = bbr_hptsi_per_second;
10309         bbr->r_ctl.bbr_hptsi_segments_delay_tar = bbr_hptsi_segments_delay_tar;
10310         bbr->r_ctl.bbr_hptsi_segments_max = bbr_hptsi_segments_max;
10311         bbr->r_ctl.bbr_hptsi_segments_floor = bbr_hptsi_segments_floor;
10312         bbr->r_ctl.bbr_hptsi_bytes_min = bbr_hptsi_bytes_min;
10313         bbr->r_ctl.bbr_cross_over = bbr_cross_over;
10314         bbr->r_ctl.rc_rtt_shrinks = cts;
10315         if (bbr->rc_use_google) {
10316                 setup_time_filter(&bbr->r_ctl.rc_delrate,
10317                                   FILTER_TYPE_MAX,
10318                                   BBR_NUM_RTTS_FOR_GOOG_DEL_LIMIT);
10319                 setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10320                                         FILTER_TYPE_MIN, (11 * USECS_IN_SECOND));
10321         } else {
10322                 setup_time_filter(&bbr->r_ctl.rc_delrate,
10323                                   FILTER_TYPE_MAX,
10324                                   bbr_num_pktepo_for_del_limit);
10325                 setup_time_filter_small(&bbr->r_ctl.rc_rttprop,
10326                                         FILTER_TYPE_MIN, (bbr_filter_len_sec * USECS_IN_SECOND));
10327         }
10328         bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_INIT, 0);
10329         if (bbr_uses_idle_restart)
10330                 bbr->rc_use_idle_restart = 1;
10331         else
10332                 bbr->rc_use_idle_restart = 0;
10333         bbr->r_ctl.rc_bbr_cur_del_rate = 0;
10334         bbr->r_ctl.rc_initial_hptsi_bw = bbr_initial_bw_bps;
10335         if (bbr_resends_use_tso)
10336                 bbr->rc_resends_use_tso = 1;
10337 #ifdef NETFLIX_PEAKRATE
10338         tp->t_peakrate_thr = tp->t_maxpeakrate;
10339 #endif
10340         if (tp->snd_una != tp->snd_max) {
10341                 /* Create a send map for the current outstanding data */
10342                 struct bbr_sendmap *rsm;
10343
10344                 rsm = bbr_alloc(bbr);
10345                 if (rsm == NULL) {
10346                         uma_zfree(bbr_pcb_zone, tp->t_fb_ptr);
10347                         tp->t_fb_ptr = NULL;
10348                         return (ENOMEM);
10349                 }
10350                 rsm->r_flags = BBR_OVERMAX;
10351                 rsm->r_tim_lastsent[0] = cts;
10352                 rsm->r_rtr_cnt = 1;
10353                 rsm->r_rtr_bytes = 0;
10354                 rsm->r_start = tp->snd_una;
10355                 rsm->r_end = tp->snd_max;
10356                 rsm->r_dupack = 0;
10357                 rsm->r_delivered = bbr->r_ctl.rc_delivered;
10358                 rsm->r_ts_valid = 0;
10359                 rsm->r_del_ack_ts = tp->ts_recent;
10360                 rsm->r_del_time = cts;
10361                 if (bbr->r_ctl.r_app_limited_until)
10362                         rsm->r_app_limited = 1;
10363                 else
10364                         rsm->r_app_limited = 0;
10365                 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_map, rsm, r_next);
10366                 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_tmap, rsm, r_tnext);
10367                 rsm->r_in_tmap = 1;
10368                 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW)
10369                         rsm->r_bbr_state = bbr_state_val(bbr);
10370                 else
10371                         rsm->r_bbr_state = 8;
10372         }
10373         if (bbr_use_rack_resend_cheat && (bbr->rc_use_google == 0))
10374                 bbr->bbr_use_rack_cheat = 1;
10375         if (bbr_incr_timers && (bbr->rc_use_google == 0))
10376                 bbr->r_ctl.rc_incr_tmrs = 1;
10377         if (bbr_include_tcp_oh && (bbr->rc_use_google == 0))
10378                 bbr->r_ctl.rc_inc_tcp_oh = 1;
10379         if (bbr_include_ip_oh && (bbr->rc_use_google == 0))
10380                 bbr->r_ctl.rc_inc_ip_oh = 1;
10381         if (bbr_include_enet_oh && (bbr->rc_use_google == 0))
10382                 bbr->r_ctl.rc_inc_enet_oh = 1;
10383
10384         bbr_log_type_statechange(bbr, cts, __LINE__);
10385         if (TCPS_HAVEESTABLISHED(tp->t_state) &&
10386             (tp->t_srtt)) {
10387                 uint32_t rtt;
10388
10389                 rtt = (TICKS_2_USEC(tp->t_srtt) >> TCP_RTT_SHIFT);
10390                 apply_filter_min_small(&bbr->r_ctl.rc_rttprop, rtt, cts);
10391         }
10392         /* announce the settings and state */
10393         bbr_log_settings_change(bbr, BBR_RECOVERY_LOWRTT);
10394         tcp_bbr_tso_size_check(bbr, cts);
10395         /*
10396          * Now call the generic function to start a timer. This will place
10397          * the TCB on the hptsi wheel if a timer is needed with appropriate
10398          * flags.
10399          */
10400         bbr_stop_all_timers(tp);
10401         bbr_start_hpts_timer(bbr, tp, cts, 5, 0, 0);
10402         return (0);
10403 }
10404
10405 /*
10406  * Return 0 if we can accept the connection. Return
10407  * non-zero if we can't handle the connection. A EAGAIN
10408  * means you need to wait until the connection is up.
10409  * a EADDRNOTAVAIL means we can never handle the connection
10410  * (no SACK).
10411  */
10412 static int
10413 bbr_handoff_ok(struct tcpcb *tp)
10414 {
10415         if ((tp->t_state == TCPS_CLOSED) ||
10416             (tp->t_state == TCPS_LISTEN)) {
10417                 /* Sure no problem though it may not stick */
10418                 return (0);
10419         }
10420         if ((tp->t_state == TCPS_SYN_SENT) ||
10421             (tp->t_state == TCPS_SYN_RECEIVED)) {
10422                 /*
10423                  * We really don't know you have to get to ESTAB or beyond
10424                  * to tell.
10425                  */
10426                 return (EAGAIN);
10427         }
10428         if ((tp->t_flags & TF_SACK_PERMIT) || bbr_sack_not_required) {
10429                 return (0);
10430         }
10431         /*
10432          * If we reach here we don't do SACK on this connection so we can
10433          * never do rack.
10434          */
10435         return (EINVAL);
10436 }
10437
10438 static void
10439 bbr_fini(struct tcpcb *tp, int32_t tcb_is_purged)
10440 {
10441         if (tp->t_fb_ptr) {
10442                 uint32_t calc;
10443                 struct tcp_bbr *bbr;
10444                 struct bbr_sendmap *rsm;
10445
10446                 bbr = (struct tcp_bbr *)tp->t_fb_ptr;
10447                 if (bbr->r_ctl.crte)
10448                         tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
10449                 bbr_log_flowend(bbr);
10450                 bbr->rc_tp = NULL;
10451                 if (tp->t_inpcb) {
10452                         /* Backout any flags2 we applied */
10453                         tp->t_inpcb->inp_flags2 &= ~INP_CANNOT_DO_ECN;
10454                         tp->t_inpcb->inp_flags2 &= ~INP_SUPPORTS_MBUFQ;
10455                         tp->t_inpcb->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
10456                 }
10457                 if (bbr->bbr_hdrw_pacing)
10458                         counter_u64_add(bbr_flows_whdwr_pacing, -1);
10459                 else
10460                         counter_u64_add(bbr_flows_nohdwr_pacing, -1);
10461                 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10462                 while (rsm) {
10463                         TAILQ_REMOVE(&bbr->r_ctl.rc_map, rsm, r_next);
10464                         uma_zfree(bbr_zone, rsm);
10465                         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
10466                 }
10467                 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10468                 while (rsm) {
10469                         TAILQ_REMOVE(&bbr->r_ctl.rc_free, rsm, r_next);
10470                         uma_zfree(bbr_zone, rsm);
10471                         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_free);
10472                 }
10473                 calc = bbr->r_ctl.rc_high_rwnd - bbr->r_ctl.rc_init_rwnd;
10474                 if (calc > (bbr->r_ctl.rc_init_rwnd / 10))
10475                         BBR_STAT_INC(bbr_dynamic_rwnd);
10476                 else
10477                         BBR_STAT_INC(bbr_static_rwnd);
10478                 bbr->r_ctl.rc_free_cnt = 0;
10479                 uma_zfree(bbr_pcb_zone, tp->t_fb_ptr);
10480                 tp->t_fb_ptr = NULL;
10481         }
10482         /* Make sure snd_nxt is correctly set */
10483         tp->snd_nxt = tp->snd_max;
10484 }
10485
10486 static void
10487 bbr_set_state(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t win)
10488 {
10489         switch (tp->t_state) {
10490         case TCPS_SYN_SENT:
10491                 bbr->r_state = TCPS_SYN_SENT;
10492                 bbr->r_substate = bbr_do_syn_sent;
10493                 break;
10494         case TCPS_SYN_RECEIVED:
10495                 bbr->r_state = TCPS_SYN_RECEIVED;
10496                 bbr->r_substate = bbr_do_syn_recv;
10497                 break;
10498         case TCPS_ESTABLISHED:
10499                 bbr->r_ctl.rc_init_rwnd = max(win, bbr->rc_tp->snd_wnd);
10500                 bbr->r_state = TCPS_ESTABLISHED;
10501                 bbr->r_substate = bbr_do_established;
10502                 break;
10503         case TCPS_CLOSE_WAIT:
10504                 bbr->r_state = TCPS_CLOSE_WAIT;
10505                 bbr->r_substate = bbr_do_close_wait;
10506                 break;
10507         case TCPS_FIN_WAIT_1:
10508                 bbr->r_state = TCPS_FIN_WAIT_1;
10509                 bbr->r_substate = bbr_do_fin_wait_1;
10510                 break;
10511         case TCPS_CLOSING:
10512                 bbr->r_state = TCPS_CLOSING;
10513                 bbr->r_substate = bbr_do_closing;
10514                 break;
10515         case TCPS_LAST_ACK:
10516                 bbr->r_state = TCPS_LAST_ACK;
10517                 bbr->r_substate = bbr_do_lastack;
10518                 break;
10519         case TCPS_FIN_WAIT_2:
10520                 bbr->r_state = TCPS_FIN_WAIT_2;
10521                 bbr->r_substate = bbr_do_fin_wait_2;
10522                 break;
10523         case TCPS_LISTEN:
10524         case TCPS_CLOSED:
10525         case TCPS_TIME_WAIT:
10526         default:
10527                 break;
10528         };
10529 }
10530
10531 static void
10532 bbr_substate_change(struct tcp_bbr *bbr, uint32_t cts, int32_t line, int dolog)
10533 {
10534         /*
10535          * Now what state are we going into now? Is there adjustments
10536          * needed?
10537          */
10538         int32_t old_state, old_gain;
10539
10540
10541         old_state = bbr_state_val(bbr);
10542         old_gain = bbr->r_ctl.rc_bbr_hptsi_gain;
10543         if (bbr_state_val(bbr) == BBR_SUB_LEVEL1) {
10544                 /* Save the lowest srtt we saw in our end of the sub-state */
10545                 bbr->rc_hit_state_1 = 0;
10546                 if (bbr->r_ctl.bbr_smallest_srtt_this_state != 0xffffffff)
10547                         bbr->r_ctl.bbr_smallest_srtt_state2 = bbr->r_ctl.bbr_smallest_srtt_this_state;
10548         }
10549         bbr->rc_bbr_substate++;
10550         if (bbr->rc_bbr_substate >= BBR_SUBSTATE_COUNT) {
10551                 /* Cycle back to first state-> gain */
10552                 bbr->rc_bbr_substate = 0;
10553         }
10554         if (bbr_state_val(bbr) == BBR_SUB_GAIN) {
10555                 /*
10556                  * We enter the gain(5/4) cycle (possibly less if
10557                  * shallow buffer detection is enabled)
10558                  */
10559                 if (bbr->skip_gain) {
10560                         /*
10561                          * Hardware pacing has set our rate to
10562                          * the max and limited our b/w just
10563                          * do level i.e. no gain.
10564                          */
10565                         bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_LEVEL1];
10566                 } else if (bbr->gain_is_limited &&
10567                            bbr->bbr_hdrw_pacing &&
10568                            bbr->r_ctl.crte) {
10569                         /*
10570                          * We can't gain above the hardware pacing
10571                          * rate which is less than our rate + the gain
10572                          * calculate the gain needed to reach the hardware
10573                          * pacing rate..
10574                          */
10575                         uint64_t bw, rate, gain_calc;
10576
10577                         bw = bbr_get_bw(bbr);
10578                         rate = bbr->r_ctl.crte->rate;
10579                         if ((rate > bw) &&
10580                             (((bw *  (uint64_t)bbr_hptsi_gain[BBR_SUB_GAIN]) / (uint64_t)BBR_UNIT) > rate)) {
10581                                 gain_calc = (rate * BBR_UNIT) / bw;
10582                                 if (gain_calc < BBR_UNIT)
10583                                         gain_calc = BBR_UNIT;
10584                                 bbr->r_ctl.rc_bbr_hptsi_gain = (uint16_t)gain_calc;
10585                         } else {
10586                                 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10587                         }
10588                 } else
10589                         bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_GAIN];
10590                 if ((bbr->rc_use_google == 0) && (bbr_gain_to_target == 0)) {
10591                         bbr->r_ctl.rc_bbr_state_atflight = cts;
10592                 } else
10593                         bbr->r_ctl.rc_bbr_state_atflight = 0;
10594         } else if (bbr_state_val(bbr) == BBR_SUB_DRAIN) {
10595                 bbr->rc_hit_state_1 = 1;
10596                 bbr->r_ctl.rc_exta_time_gd = 0;
10597                 bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10598                                                      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10599                 if (bbr_state_drain_2_tar) {
10600                         bbr->r_ctl.rc_bbr_state_atflight = 0;
10601                 } else
10602                         bbr->r_ctl.rc_bbr_state_atflight = cts;
10603                 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[BBR_SUB_DRAIN];
10604         } else {
10605                 /* All other cycles hit here 2-7 */
10606                 if ((old_state == BBR_SUB_DRAIN) && bbr->rc_hit_state_1) {
10607                         if (bbr_sub_drain_slam_cwnd &&
10608                             (bbr->rc_use_google == 0) &&
10609                             (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10610                                 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10611                                 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10612                         }
10613                         if ((cts - bbr->r_ctl.rc_bbr_state_time) > bbr_get_rtt(bbr, BBR_RTT_PROP))
10614                                 bbr->r_ctl.rc_exta_time_gd += ((cts - bbr->r_ctl.rc_bbr_state_time) -
10615                                                                bbr_get_rtt(bbr, BBR_RTT_PROP));
10616                         else
10617                                 bbr->r_ctl.rc_exta_time_gd = 0;
10618                         if (bbr->r_ctl.rc_exta_time_gd) {
10619                                 bbr->r_ctl.rc_level_state_extra = bbr->r_ctl.rc_exta_time_gd;
10620                                 /* Now chop up the time for each state (div by 7) */
10621                                 bbr->r_ctl.rc_level_state_extra /= 7;
10622                                 if (bbr_rand_ot && bbr->r_ctl.rc_level_state_extra) {
10623                                         /* Add a randomization */
10624                                         bbr_randomize_extra_state_time(bbr);
10625                                 }
10626                         }
10627                 }
10628                 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10629                 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_hptsi_gain[bbr_state_val(bbr)];
10630         }
10631         if (bbr->rc_use_google) {
10632                 bbr->r_ctl.rc_bbr_state_atflight = max(1, cts);
10633         }
10634         bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10635         bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
10636         if (dolog)
10637                 bbr_log_type_statechange(bbr, cts, line);
10638
10639         if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10640                 uint32_t time_in;
10641
10642                 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10643                 if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
10644                         counter_u64_add(bbr_state_time[(old_state + 5)], time_in);
10645                 } else {
10646                         counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10647                 }
10648         }
10649         bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
10650         bbr_set_state_target(bbr, __LINE__);
10651         if (bbr_sub_drain_slam_cwnd &&
10652             (bbr->rc_use_google == 0) &&
10653             (bbr_state_val(bbr) == BBR_SUB_DRAIN)) {
10654                 /* Slam down the cwnd */
10655                 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10656                 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10657                 if (bbr_sub_drain_app_limit) {
10658                         /* Go app limited if we are on a long drain */
10659                         bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered +
10660                                                           ctf_flight_size(bbr->rc_tp,
10661                                                               (bbr->r_ctl.rc_sacked +
10662                                                                bbr->r_ctl.rc_lost_bytes)));
10663                 }
10664                 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10665         }
10666         if (bbr->rc_lt_use_bw) {
10667                 /* In policed mode we clamp pacing_gain to BBR_UNIT */
10668                 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10669         }
10670         /* Google changes TSO size every cycle */
10671         if (bbr->rc_use_google)
10672                 tcp_bbr_tso_size_check(bbr, cts);
10673         bbr->r_ctl.gain_epoch = cts;
10674         bbr->r_ctl.rc_bbr_state_time = cts;
10675         bbr->r_ctl.substate_pe = bbr->r_ctl.rc_pkt_epoch;
10676 }
10677
10678 static void
10679 bbr_set_probebw_google_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
10680 {
10681         if ((bbr_state_val(bbr) == BBR_SUB_DRAIN) &&
10682             (google_allow_early_out == 1) &&
10683             (bbr->r_ctl.rc_flight_at_input <= bbr->r_ctl.rc_target_at_state)) {
10684                 /* We have reached out target flight size possibly early */
10685                 goto change_state;
10686         }
10687         if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10688                 return;
10689         }
10690         if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_get_rtt(bbr, BBR_RTT_PROP)) {
10691                 /*
10692                  * Must be a rttProp movement forward before
10693                  * we can change states.
10694                  */
10695                 return;
10696         }
10697         if (bbr_state_val(bbr) == BBR_SUB_GAIN) {
10698                 /*
10699                  * The needed time has passed but for
10700                  * the gain cycle extra rules apply:
10701                  * 1) If we have seen loss, we exit
10702                  * 2) If we have not reached the target
10703                  *    we stay in GAIN (gain-to-target).
10704                  */
10705                 if (google_consider_lost && losses)
10706                         goto change_state;
10707                 if (bbr->r_ctl.rc_target_at_state > bbr->r_ctl.rc_flight_at_input) {
10708                         return;
10709                 }
10710         }
10711 change_state:
10712         /* For gain we must reach our target, all others last 1 rttProp */
10713         bbr_substate_change(bbr, cts, __LINE__, 1);
10714 }
10715
10716 static void
10717 bbr_set_probebw_gains(struct tcp_bbr *bbr, uint32_t cts, uint32_t losses)
10718 {
10719         uint32_t flight, bbr_cur_cycle_time;
10720
10721         if (bbr->rc_use_google) {
10722                 bbr_set_probebw_google_gains(bbr, cts, losses);
10723                 return;
10724         }
10725         if (cts == 0) {
10726                 /*
10727                  * Never alow cts to be 0 we
10728                  * do this so we can judge if
10729                  * we have set a timestamp.
10730                  */
10731                 cts = 1;
10732         }
10733         if (bbr_state_is_pkt_epoch)
10734                 bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PKTRTT);
10735         else
10736                 bbr_cur_cycle_time = bbr_get_rtt(bbr, BBR_RTT_PROP);
10737
10738         if (bbr->r_ctl.rc_bbr_state_atflight == 0) {
10739                 if (bbr_state_val(bbr) == BBR_SUB_DRAIN) {
10740                         flight = ctf_flight_size(bbr->rc_tp,
10741                                      (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10742                         if (bbr_sub_drain_slam_cwnd && bbr->rc_hit_state_1) {
10743                                 /* Keep it slam down */
10744                                 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state) {
10745                                         bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10746                                         bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10747                                 }
10748                                 if (bbr_sub_drain_app_limit) {
10749                                         /* Go app limited if we are on a long drain */
10750                                         bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.rc_delivered + flight);
10751                                 }
10752                         }
10753                         if (TSTMP_GT(cts, bbr->r_ctl.gain_epoch) &&
10754                             (((cts - bbr->r_ctl.gain_epoch) > bbr_get_rtt(bbr, BBR_RTT_PROP)) ||
10755                              (flight >= bbr->r_ctl.flightsize_at_drain))) {
10756                                 /*
10757                                  * Still here after the same time as
10758                                  * the gain. We need to drain harder
10759                                  * for the next srtt. Reduce by a set amount
10760                                  * the gain drop is capped at DRAIN states
10761                                  * value (88).
10762                                  */
10763                                 bbr->r_ctl.flightsize_at_drain = flight;
10764                                 if (bbr_drain_drop_mul &&
10765                                     bbr_drain_drop_div &&
10766                                     (bbr_drain_drop_mul < bbr_drain_drop_div)) {
10767                                         /* Use your specific drop value (def 4/5 = 20%) */
10768                                         bbr->r_ctl.rc_bbr_hptsi_gain *= bbr_drain_drop_mul;
10769                                         bbr->r_ctl.rc_bbr_hptsi_gain /= bbr_drain_drop_div;
10770                                 } else {
10771                                         /* You get drop of 20% */
10772                                         bbr->r_ctl.rc_bbr_hptsi_gain *= 4;
10773                                         bbr->r_ctl.rc_bbr_hptsi_gain /= 5;
10774                                 }
10775                                 if (bbr->r_ctl.rc_bbr_hptsi_gain <= bbr_drain_floor) {
10776                                         /* Reduce our gain again to the bottom  */
10777                                         bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
10778                                 }
10779                                 bbr_log_exit_gain(bbr, cts, 4);
10780                                 /*
10781                                  * Extend out so we wait another
10782                                  * epoch before dropping again.
10783                                  */
10784                                 bbr->r_ctl.gain_epoch = cts;
10785                         }
10786                         if (flight <= bbr->r_ctl.rc_target_at_state) {
10787                                 if (bbr_sub_drain_slam_cwnd &&
10788                                     (bbr->rc_use_google == 0) &&
10789                                     (bbr->rc_tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
10790                                         bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
10791                                         bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10792                                 }
10793                                 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10794                                 bbr_log_exit_gain(bbr, cts, 3);
10795                         }
10796                 } else {
10797                         /* Its a gain  */
10798                         if (bbr->r_ctl.rc_lost > bbr->r_ctl.bbr_lost_at_state) {
10799                                 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10800                                 goto change_state;
10801                         }
10802                         if ((ctf_outstanding(bbr->rc_tp) >= bbr->r_ctl.rc_target_at_state) ||
10803                             ((ctf_outstanding(bbr->rc_tp) +  bbr->rc_tp->t_maxseg - 1) >=
10804                              bbr->rc_tp->snd_wnd)) {
10805                                 bbr->r_ctl.rc_bbr_state_atflight = max(cts, 1);
10806                                 bbr_log_exit_gain(bbr, cts, 2);
10807                         }
10808                 }
10809                 /**
10810                  * We fall through and return always one of two things has
10811                  * occured.
10812                  * 1) We are still not at target
10813                  *    <or>
10814                  * 2) We reached the target and set rc_bbr_state_atflight
10815                  *    which means we no longer hit this block
10816                  *    next time we are called.
10817                  */
10818                 return;
10819         }
10820 change_state:
10821         if (TSTMP_LT(cts, bbr->r_ctl.rc_bbr_state_time))
10822                 return;
10823         if ((cts - bbr->r_ctl.rc_bbr_state_time) < bbr_cur_cycle_time) {
10824                 /* Less than a full time-period has passed */
10825                 return;
10826         }
10827         if (bbr->r_ctl.rc_level_state_extra &&
10828             (bbr_state_val(bbr) > BBR_SUB_DRAIN) &&
10829             ((cts - bbr->r_ctl.rc_bbr_state_time) <
10830              (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10831                 /* Less than a full time-period + extra has passed */
10832                 return;
10833         }
10834         if (bbr_gain_gets_extra_too &&
10835             bbr->r_ctl.rc_level_state_extra &&
10836             (bbr_state_val(bbr) == BBR_SUB_GAIN) &&
10837             ((cts - bbr->r_ctl.rc_bbr_state_time) <
10838              (bbr_cur_cycle_time + bbr->r_ctl.rc_level_state_extra))) {
10839                 /* Less than a full time-period + extra has passed */
10840                 return;
10841         }
10842         bbr_substate_change(bbr, cts, __LINE__, 1);
10843 }
10844
10845 static uint32_t
10846 bbr_get_a_state_target(struct tcp_bbr *bbr, uint32_t gain)
10847 {
10848         uint32_t mss, tar;
10849
10850         if (bbr->rc_use_google) {
10851                 /* Google just uses the cwnd target */
10852                 tar = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), gain);
10853         } else {
10854                 mss = min((bbr->rc_tp->t_maxseg - bbr->rc_last_options),
10855                           bbr->r_ctl.rc_pace_max_segs);
10856                 /* Get the base cwnd with gain rounded to a mss */
10857                 tar = roundup(bbr_get_raw_target_cwnd(bbr, bbr_get_bw(bbr),
10858                                                       gain), mss);
10859                 /* Make sure it is within our min */
10860                 if (tar < get_min_cwnd(bbr))
10861                         return (get_min_cwnd(bbr));
10862         }
10863         return (tar);
10864 }
10865
10866 static void
10867 bbr_set_state_target(struct tcp_bbr *bbr, int line)
10868 {
10869         uint32_t tar, meth;
10870
10871         if ((bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) &&
10872             ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google)) {
10873                 /* Special case using old probe-rtt method */
10874                 tar = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10875                 meth = 1;
10876         } else {
10877                 /* Non-probe-rtt case and reduced probe-rtt  */
10878                 if ((bbr->rc_bbr_state == BBR_STATE_PROBE_BW) &&
10879                     (bbr->r_ctl.rc_bbr_hptsi_gain > BBR_UNIT)) {
10880                         /* For gain cycle we use the hptsi gain */
10881                         tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10882                         meth = 2;
10883                 } else if ((bbr_target_is_bbunit) || bbr->rc_use_google) {
10884                         /*
10885                          * If configured, or for google all other states
10886                          * get BBR_UNIT.
10887                          */
10888                         tar = bbr_get_a_state_target(bbr, BBR_UNIT);
10889                         meth = 3;
10890                 } else {
10891                         /*
10892                          * Or we set a target based on the pacing gain
10893                          * for non-google mode and default (non-configured).
10894                          * Note we don't set a target goal below drain (192).
10895                          */
10896                         if (bbr->r_ctl.rc_bbr_hptsi_gain < bbr_hptsi_gain[BBR_SUB_DRAIN])  {
10897                                 tar = bbr_get_a_state_target(bbr, bbr_hptsi_gain[BBR_SUB_DRAIN]);
10898                                 meth = 4;
10899                         } else {
10900                                 tar = bbr_get_a_state_target(bbr, bbr->r_ctl.rc_bbr_hptsi_gain);
10901                                 meth = 5;
10902                         }
10903                 }
10904         }
10905         bbr_log_set_of_state_target(bbr, tar, line, meth);
10906         bbr->r_ctl.rc_target_at_state = tar;
10907 }
10908
10909 static void
10910 bbr_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts, int32_t line)
10911 {
10912         /* Change to probe_rtt */
10913         uint32_t time_in;
10914
10915         bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
10916         bbr->r_ctl.flightsize_at_drain = ctf_flight_size(bbr->rc_tp,
10917                                              (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
10918         bbr->r_ctl.r_app_limited_until = (bbr->r_ctl.flightsize_at_drain
10919                                           + bbr->r_ctl.rc_delivered);
10920         /* Setup so we force feed the filter */
10921         if (bbr->rc_use_google || bbr_probertt_sets_rtt)
10922                 bbr->rc_prtt_set_ts = 1;
10923         if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
10924                 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
10925                 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
10926         }
10927         bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_ENTERPROBE, 0);
10928         bbr->r_ctl.rc_rtt_shrinks = cts;
10929         bbr->r_ctl.last_in_probertt = cts;
10930         bbr->r_ctl.rc_probertt_srttchktim = cts;
10931         bbr->r_ctl.rc_bbr_state_time = cts;
10932         bbr->rc_bbr_state = BBR_STATE_PROBE_RTT;
10933         /* We need to force the filter to update */
10934
10935         if ((bbr_sub_drain_slam_cwnd) &&
10936             bbr->rc_hit_state_1 &&
10937             (bbr->rc_use_google == 0) &&
10938             (bbr_state_val(bbr) == BBR_SUB_DRAIN)) {
10939                 if (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_saved_cwnd)
10940                         bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10941         } else
10942                 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
10943         /* Update the lost */
10944         bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
10945         if ((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google){
10946                 /* Set to the non-configurable default of 4 (PROBE_RTT_MIN)  */
10947                 bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
10948                 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10949                 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
10950                 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10951                 bbr_log_set_of_state_target(bbr, bbr->rc_tp->snd_cwnd, __LINE__, 6);
10952                 bbr->r_ctl.rc_target_at_state = bbr->rc_tp->snd_cwnd;
10953         } else {
10954                 /*
10955                  * We bring it down slowly by using a hptsi gain that is
10956                  * probably 75%. This will slowly float down our outstanding
10957                  * without tampering with the cwnd.
10958                  */
10959                 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
10960                 bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
10961                 bbr_set_state_target(bbr, __LINE__);
10962                 if (bbr_prtt_slam_cwnd &&
10963                     (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
10964                         bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
10965                         bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
10966                 }
10967         }
10968         if (ctf_flight_size(bbr->rc_tp,
10969                 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
10970             bbr->r_ctl.rc_target_at_state) {
10971                 /* We are at target */
10972                 bbr->r_ctl.rc_bbr_enters_probertt = cts;
10973         } else {
10974                 /* We need to come down to reach target before our time begins */
10975                 bbr->r_ctl.rc_bbr_enters_probertt = 0;
10976         }
10977         bbr->r_ctl.rc_pe_of_prtt = bbr->r_ctl.rc_pkt_epoch;
10978         BBR_STAT_INC(bbr_enter_probertt);
10979         bbr_log_exit_gain(bbr, cts, 0);
10980         bbr_log_type_statechange(bbr, cts, line);
10981 }
10982
10983 static void
10984 bbr_check_probe_rtt_limits(struct tcp_bbr *bbr, uint32_t cts)
10985 {
10986         /*
10987          * Sanity check on probe-rtt intervals.
10988          * In crazy situations where we are competing
10989          * against new-reno flows with huge buffers
10990          * our rtt-prop interval could come to dominate
10991          * things if we can't get through a full set
10992          * of cycles, we need to adjust it.
10993          */
10994         if (bbr_can_adjust_probertt &&
10995             (bbr->rc_use_google == 0)) {
10996                 uint16_t val = 0;
10997                 uint32_t cur_rttp, fval, newval, baseval;
10998
10999                 /* Are we to small and go into probe-rtt to often? */
11000                 baseval = (bbr_get_rtt(bbr, BBR_RTT_PROP) * (BBR_SUBSTATE_COUNT + 1));
11001                 cur_rttp = roundup(baseval, USECS_IN_SECOND);
11002                 fval = bbr_filter_len_sec * USECS_IN_SECOND;
11003                 if (bbr_is_ratio == 0) {
11004                         if (fval > bbr_rtt_probe_limit)
11005                                 newval = cur_rttp + (fval - bbr_rtt_probe_limit);
11006                         else
11007                                 newval = cur_rttp;
11008                 } else {
11009                         int mul;
11010
11011                         mul = fval / bbr_rtt_probe_limit;
11012                         newval = cur_rttp * mul;
11013                 }
11014                 if (cur_rttp >  bbr->r_ctl.rc_probertt_int) {
11015                         bbr->r_ctl.rc_probertt_int = cur_rttp;
11016                         reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
11017                         val = 1;
11018                 } else {
11019                         /*
11020                          * No adjustments were made
11021                          * do we need to shrink it?
11022                          */
11023                         if (bbr->r_ctl.rc_probertt_int > bbr_rtt_probe_limit) {
11024                                 if (cur_rttp <= bbr_rtt_probe_limit) {
11025                                         /*
11026                                          * Things have calmed down lets
11027                                          * shrink all the way to default
11028                                          */
11029                                         bbr->r_ctl.rc_probertt_int = bbr_rtt_probe_limit;
11030                                         reset_time_small(&bbr->r_ctl.rc_rttprop,
11031                                                          (bbr_filter_len_sec * USECS_IN_SECOND));
11032                                         cur_rttp = bbr_rtt_probe_limit;
11033                                         newval = (bbr_filter_len_sec * USECS_IN_SECOND);
11034                                         val = 2;
11035                                 } else {
11036                                         /*
11037                                          * Well does some adjustment make sense?
11038                                          */
11039                                         if (cur_rttp < bbr->r_ctl.rc_probertt_int) {
11040                                                 /* We can reduce interval time some */
11041                                                 bbr->r_ctl.rc_probertt_int = cur_rttp;
11042                                                 reset_time_small(&bbr->r_ctl.rc_rttprop, newval);
11043                                                 val = 3;
11044                                         }
11045                                 }
11046                         }
11047                 }
11048                 if (val)
11049                         bbr_log_rtt_shrinks(bbr, cts, cur_rttp, newval, __LINE__, BBR_RTTS_RESETS_VALUES, val);
11050         }
11051 }
11052
11053 static void
11054 bbr_exit_probe_rtt(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t cts)
11055 {
11056         /* Exit probe-rtt */
11057
11058         if (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd) {
11059                 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
11060                 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11061         }
11062         bbr_log_exit_gain(bbr, cts, 1);
11063         bbr->rc_hit_state_1 = 0;
11064         bbr->r_ctl.rc_rtt_shrinks = cts;
11065         bbr->r_ctl.last_in_probertt = cts;
11066         bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_RTTPROBE, 0);
11067         bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11068         bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp,
11069                                               (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
11070                                           bbr->r_ctl.rc_delivered);
11071         if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11072                 uint32_t time_in;
11073
11074                 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11075                 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11076         }
11077         if (bbr->rc_filled_pipe) {
11078                 /* Switch to probe_bw */
11079                 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11080                 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11081                 bbr->r_ctl.rc_bbr_cwnd_gain = bbr_cwnd_gain;
11082                 bbr_substate_change(bbr, cts, __LINE__, 0);
11083                 bbr_log_type_statechange(bbr, cts, __LINE__);
11084         } else {
11085                 /* Back to startup */
11086                 bbr->rc_bbr_state = BBR_STATE_STARTUP;
11087                 bbr->r_ctl.rc_bbr_state_time = cts;
11088                 /*
11089                  * We don't want to give a complete free 3
11090                  * measurements until we exit, so we use
11091                  * the number of pe's we were in probe-rtt
11092                  * to add to the startup_epoch. That way
11093                  * we will still retain the old state.
11094                  */
11095                 bbr->r_ctl.rc_bbr_last_startup_epoch += (bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_pe_of_prtt);
11096                 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11097                 /* Make sure to use the lower pg when shifting back in */
11098                 if (bbr->r_ctl.rc_lost &&
11099                     bbr_use_lower_gain_in_startup &&
11100                     (bbr->rc_use_google == 0))
11101                         bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
11102                 else
11103                         bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_startup_pg;
11104                 bbr->r_ctl.rc_bbr_cwnd_gain = bbr->r_ctl.rc_startup_pg;
11105                 /* Probably not needed but set it anyway */
11106                 bbr_set_state_target(bbr, __LINE__);
11107                 bbr_log_type_statechange(bbr, cts, __LINE__);
11108                 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11109                     bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 0);
11110         }
11111         bbr_check_probe_rtt_limits(bbr, cts);
11112 }
11113
11114 static int32_t inline
11115 bbr_should_enter_probe_rtt(struct tcp_bbr *bbr, uint32_t cts)
11116 {
11117         if ((bbr->rc_past_init_win == 1) &&
11118             (bbr->rc_in_persist == 0) &&
11119             (bbr_calc_time(cts, bbr->r_ctl.rc_rtt_shrinks) >= bbr->r_ctl.rc_probertt_int)) {
11120                 return (1);
11121         }
11122         if (bbr_can_force_probertt &&
11123             (bbr->rc_in_persist == 0) &&
11124             (TSTMP_GT(cts, bbr->r_ctl.last_in_probertt)) &&
11125             ((cts - bbr->r_ctl.last_in_probertt) > bbr->r_ctl.rc_probertt_int)) {
11126                 return (1);
11127         }
11128         return (0);
11129 }
11130
11131
11132 static int32_t
11133 bbr_google_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t  pkt_epoch)
11134 {
11135         uint64_t btlbw, gain;
11136         if (pkt_epoch == 0) {
11137                 /*
11138                  * Need to be on a pkt-epoch to continue.
11139                  */
11140                 return (0);
11141         }
11142         btlbw = bbr_get_full_bw(bbr);
11143         gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
11144                  (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
11145         if (btlbw >= gain) {
11146                 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
11147                 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11148                                       bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
11149                 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
11150         }
11151         if ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)
11152                 return (1);
11153         bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11154                               bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
11155         return(0);
11156 }
11157
11158 static int32_t inline
11159 bbr_state_startup(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch)
11160 {
11161         /* Have we gained 25% in the last 3 packet based epoch's? */
11162         uint64_t btlbw, gain;
11163         int do_exit;
11164         int delta, rtt_gain;
11165
11166         if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
11167             (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
11168                 /*
11169                  * This qualifies as a RTT_PROBE session since we drop the
11170                  * data outstanding to nothing and waited more than
11171                  * bbr_rtt_probe_time.
11172                  */
11173                 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
11174                 bbr_set_reduced_rtt(bbr, cts, __LINE__);
11175         }
11176         if (bbr_should_enter_probe_rtt(bbr, cts)) {
11177                 bbr_enter_probe_rtt(bbr, cts, __LINE__);
11178                 return (0);
11179         }
11180         if (bbr->rc_use_google)
11181                 return (bbr_google_startup(bbr, cts,  pkt_epoch));
11182
11183         if ((bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
11184             (bbr_use_lower_gain_in_startup)) {
11185                 /* Drop to a lower gain 1.5 x since we saw loss */
11186                 bbr->r_ctl.rc_bbr_hptsi_gain = bbr_startup_lower;
11187         }
11188         if (pkt_epoch == 0) {
11189                 /*
11190                  * Need to be on a pkt-epoch to continue.
11191                  */
11192                 return (0);
11193         }
11194         if (bbr_rtt_gain_thresh) {
11195                 /*
11196                  * Do we allow a flow to stay
11197                  * in startup with no loss and no
11198                  * gain in rtt over a set threshold?
11199                  */
11200                 if (bbr->r_ctl.rc_pkt_epoch_rtt &&
11201                     bbr->r_ctl.startup_last_srtt &&
11202                     (bbr->r_ctl.rc_pkt_epoch_rtt > bbr->r_ctl.startup_last_srtt)) {
11203                         delta = bbr->r_ctl.rc_pkt_epoch_rtt - bbr->r_ctl.startup_last_srtt;
11204                         rtt_gain = (delta * 100) / bbr->r_ctl.startup_last_srtt;
11205                 } else
11206                         rtt_gain = 0;
11207                 if ((bbr->r_ctl.startup_last_srtt == 0)  ||
11208                     (bbr->r_ctl.rc_pkt_epoch_rtt < bbr->r_ctl.startup_last_srtt))
11209                         /* First time or new lower value */
11210                         bbr->r_ctl.startup_last_srtt = bbr->r_ctl.rc_pkt_epoch_rtt;
11211
11212                 if ((bbr->r_ctl.rc_lost == 0) &&
11213                     (rtt_gain < bbr_rtt_gain_thresh)) {
11214                         /*
11215                          * No loss, and we are under
11216                          * our gain threhold for
11217                          * increasing RTT.
11218                          */
11219                         if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
11220                                 bbr->r_ctl.rc_bbr_last_startup_epoch++;
11221                         bbr_log_startup_event(bbr, cts, rtt_gain,
11222                                               delta, bbr->r_ctl.startup_last_srtt, 10);
11223                         return (0);
11224                 }
11225         }
11226         if ((bbr->r_ctl.r_measurement_count == bbr->r_ctl.last_startup_measure) &&
11227             (bbr->r_ctl.rc_lost_at_startup == bbr->r_ctl.rc_lost) &&
11228             (!IN_RECOVERY(bbr->rc_tp->t_flags))) {
11229                 /*
11230                  * We only assess if we have a new measurment when
11231                  * we have no loss and are not in recovery.
11232                  * Drag up by one our last_startup epoch so we will hold
11233                  * the number of non-gain we have already accumulated.
11234                  */
11235                 if (bbr->r_ctl.rc_bbr_last_startup_epoch < bbr->r_ctl.rc_pkt_epoch)
11236                         bbr->r_ctl.rc_bbr_last_startup_epoch++;
11237                 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11238                                       bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 9);
11239                 return (0);
11240         }
11241         /* Case where we reduced the lost (bad retransmit) */
11242         if (bbr->r_ctl.rc_lost_at_startup > bbr->r_ctl.rc_lost)
11243                 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11244         bbr->r_ctl.last_startup_measure = bbr->r_ctl.r_measurement_count;
11245         btlbw = bbr_get_full_bw(bbr);
11246         if (bbr->r_ctl.rc_bbr_hptsi_gain == bbr_startup_lower)
11247                 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
11248                          (uint64_t)bbr_low_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
11249         else
11250                 gain = ((bbr->r_ctl.rc_bbr_lastbtlbw *
11251                          (uint64_t)bbr_start_exit) / (uint64_t)100) + bbr->r_ctl.rc_bbr_lastbtlbw;
11252         do_exit = 0;
11253         if (btlbw > bbr->r_ctl.rc_bbr_lastbtlbw)
11254                 bbr->r_ctl.rc_bbr_lastbtlbw = btlbw;
11255         if (btlbw >= gain) {
11256                 bbr->r_ctl.rc_bbr_last_startup_epoch = bbr->r_ctl.rc_pkt_epoch;
11257                 /* Update the lost so we won't exit in next set of tests */
11258                 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11259                 bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11260                                       bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 3);
11261         }
11262         if ((bbr->rc_loss_exit &&
11263              (bbr->r_ctl.rc_lost > bbr->r_ctl.rc_lost_at_startup) &&
11264              (bbr->r_ctl.rc_pkt_epoch_loss_rate > bbr_startup_loss_thresh)) &&
11265             ((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS)) {
11266                 /*
11267                  * If we had no gain,  we had loss and that loss was above
11268                  * our threshould, the rwnd is not constrained, and we have
11269                  * had at least 3 packet epochs exit. Note that this is
11270                  * switched off by sysctl. Google does not do this by the
11271                  * way.
11272                  */
11273                 if ((ctf_flight_size(bbr->rc_tp,
11274                          (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) +
11275                      (2 * max(bbr->r_ctl.rc_pace_max_segs, bbr->rc_tp->t_maxseg))) <= bbr->rc_tp->snd_wnd) {
11276                         do_exit = 1;
11277                         bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11278                                               bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 4);
11279                 } else {
11280                         /* Just record an updated loss value */
11281                         bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11282                         bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11283                                               bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 5);
11284                 }
11285         } else
11286                 bbr->r_ctl.rc_lost_at_startup = bbr->r_ctl.rc_lost;
11287         if (((bbr->r_ctl.rc_pkt_epoch - bbr->r_ctl.rc_bbr_last_startup_epoch) >= BBR_STARTUP_EPOCHS) ||
11288             do_exit) {
11289                 /* Return 1 to exit the startup state. */
11290                 return (1);
11291         }
11292         /* Stay in startup */
11293         bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11294                               bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 8);
11295         return (0);
11296 }
11297
11298 static void
11299 bbr_state_change(struct tcp_bbr *bbr, uint32_t cts, int32_t epoch, int32_t pkt_epoch, uint32_t losses)
11300 {
11301         /*
11302          * A tick occured in the rtt epoch do we need to do anything?
11303          */
11304 #ifdef BBR_INVARIANTS
11305         if ((bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
11306             (bbr->rc_bbr_state != BBR_STATE_DRAIN) &&
11307             (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) &&
11308             (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
11309             (bbr->rc_bbr_state != BBR_STATE_PROBE_BW)) {
11310                 /* Debug code? */
11311                 panic("Unknown BBR state %d?\n", bbr->rc_bbr_state);
11312         }
11313 #endif
11314         if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
11315                 /* Do we exit the startup state? */
11316                 if (bbr_state_startup(bbr, cts, epoch, pkt_epoch)) {
11317                         uint32_t time_in;
11318
11319                         bbr_log_startup_event(bbr, cts, bbr->r_ctl.rc_bbr_last_startup_epoch,
11320                                               bbr->r_ctl.rc_lost_at_startup, bbr_start_exit, 6);
11321                         bbr->rc_filled_pipe = 1;
11322                         bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11323                         if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11324
11325                                 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11326                                 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11327                         } else
11328                                 time_in = 0;
11329                         if (bbr->rc_no_pacing)
11330                                 bbr->rc_no_pacing = 0;
11331                         bbr->r_ctl.rc_bbr_state_time = cts;
11332                         bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.rc_drain_pg;
11333                         bbr->rc_bbr_state = BBR_STATE_DRAIN;
11334                         bbr_set_state_target(bbr, __LINE__);
11335                         if ((bbr->rc_use_google == 0) &&
11336                             bbr_slam_cwnd_in_main_drain) {
11337                                 /* Here we don't have to worry about probe-rtt */
11338                                 bbr->r_ctl.rc_saved_cwnd = bbr->rc_tp->snd_cwnd;
11339                                 bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11340                                 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11341                         }
11342                         bbr->r_ctl.rc_bbr_cwnd_gain = bbr_high_gain;
11343                         bbr_log_type_statechange(bbr, cts, __LINE__);
11344                         if (ctf_flight_size(bbr->rc_tp,
11345                                 (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes)) <=
11346                             bbr->r_ctl.rc_target_at_state) {
11347                                 /*
11348                                  * Switch to probe_bw if we are already
11349                                  * there
11350                                  */
11351                                 bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11352                                 bbr_substate_change(bbr, cts, __LINE__, 0);
11353                                 bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11354                                 bbr_log_type_statechange(bbr, cts, __LINE__);
11355                         }
11356                 }
11357         } else if (bbr->rc_bbr_state == BBR_STATE_IDLE_EXIT) {
11358                 uint32_t inflight;
11359                 struct tcpcb *tp;
11360
11361                 tp = bbr->rc_tp;
11362                 inflight = ctf_flight_size(tp,
11363                               (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11364                 if (inflight >= bbr->r_ctl.rc_target_at_state) {
11365                         /* We have reached a flight of the cwnd target */
11366                         bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11367                         bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11368                         bbr->r_ctl.rc_bbr_cwnd_gain = BBR_UNIT;
11369                         bbr_set_state_target(bbr, __LINE__);
11370                         /*
11371                          * Rig it so we don't do anything crazy and
11372                          * start fresh with a new randomization.
11373                          */
11374                         bbr->r_ctl.bbr_smallest_srtt_this_state = 0xffffffff;
11375                         bbr->rc_bbr_substate = BBR_SUB_LEVEL6;
11376                         bbr_substate_change(bbr, cts, __LINE__, 1);
11377                 }
11378         } else if (bbr->rc_bbr_state == BBR_STATE_DRAIN) {
11379                 /* Has in-flight reached the bdp (or less)? */
11380                 uint32_t inflight;
11381                 struct tcpcb *tp;
11382
11383                 tp = bbr->rc_tp;
11384                 inflight = ctf_flight_size(tp,
11385                               (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11386                 if ((bbr->rc_use_google == 0) &&
11387                     bbr_slam_cwnd_in_main_drain &&
11388                     (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11389                         /*
11390                          * Here we don't have to worry about probe-rtt
11391                          * re-slam it, but keep it slammed down.
11392                          */
11393                         bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11394                         bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11395                 }
11396                 if (inflight <= bbr->r_ctl.rc_target_at_state) {
11397                         /* We have drained */
11398                         bbr->rc_bbr_state = BBR_STATE_PROBE_BW;
11399                         bbr->r_ctl.bbr_lost_at_state = bbr->r_ctl.rc_lost;
11400                         if (SEQ_GT(cts, bbr->r_ctl.rc_bbr_state_time)) {
11401                                 uint32_t time_in;
11402
11403                                 time_in = cts - bbr->r_ctl.rc_bbr_state_time;
11404                                 counter_u64_add(bbr_state_time[bbr->rc_bbr_state], time_in);
11405                         }
11406                         if ((bbr->rc_use_google == 0) &&
11407                             bbr_slam_cwnd_in_main_drain &&
11408                             (tp->snd_cwnd < bbr->r_ctl.rc_saved_cwnd)) {
11409                                 /* Restore the cwnd */
11410                                 tp->snd_cwnd = bbr->r_ctl.rc_saved_cwnd;
11411                                 bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11412                         }
11413                         /* Setup probe-rtt has being done now RRS-HERE */
11414                         bbr->r_ctl.rc_rtt_shrinks = cts;
11415                         bbr->r_ctl.last_in_probertt = cts;
11416                         bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_LEAVE_DRAIN, 0);
11417                         /* Randomly pick a sub-state */
11418                         bbr->rc_bbr_substate = bbr_pick_probebw_substate(bbr, cts);
11419                         bbr_substate_change(bbr, cts, __LINE__, 0);
11420                         bbr_log_type_statechange(bbr, cts, __LINE__);
11421                 }
11422         } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_RTT) {
11423                 uint32_t flight;
11424
11425                 flight = ctf_flight_size(bbr->rc_tp,
11426                              (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11427                 bbr->r_ctl.r_app_limited_until = (flight + bbr->r_ctl.rc_delivered);
11428                 if (((bbr->r_ctl.bbr_rttprobe_gain_val == 0) || bbr->rc_use_google) &&
11429                     (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11430                         /*
11431                          * We must keep cwnd at the desired MSS.
11432                          */
11433                         bbr->rc_tp->snd_cwnd = bbr_rtt_probe_cwndtarg * (bbr->rc_tp->t_maxseg - bbr->rc_last_options);
11434                         bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11435                 } else if ((bbr_prtt_slam_cwnd) &&
11436                            (bbr->rc_tp->snd_cwnd > bbr->r_ctl.rc_target_at_state)) {
11437                         /* Re-slam it */
11438                         bbr->rc_tp->snd_cwnd = bbr->r_ctl.rc_target_at_state;
11439                         bbr_log_type_cwndupd(bbr, 0, 0, 0, 12, 0, 0, __LINE__);
11440                 }
11441                 if (bbr->r_ctl.rc_bbr_enters_probertt == 0) {
11442                         /* Has outstanding reached our target? */
11443                         if (flight <= bbr->r_ctl.rc_target_at_state) {
11444                                 bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_REACHTAR, 0);
11445                                 bbr->r_ctl.rc_bbr_enters_probertt = cts;
11446                                 /* If time is exactly 0, be 1usec off */
11447                                 if (bbr->r_ctl.rc_bbr_enters_probertt == 0)
11448                                         bbr->r_ctl.rc_bbr_enters_probertt = 1;
11449                                 if (bbr->rc_use_google == 0) {
11450                                         /*
11451                                          * Restore any lowering that as occured to
11452                                          * reach here
11453                                          */
11454                                         if (bbr->r_ctl.bbr_rttprobe_gain_val)
11455                                                 bbr->r_ctl.rc_bbr_hptsi_gain = bbr->r_ctl.bbr_rttprobe_gain_val;
11456                                         else
11457                                                 bbr->r_ctl.rc_bbr_hptsi_gain = BBR_UNIT;
11458                                 }
11459                         }
11460                         if ((bbr->r_ctl.rc_bbr_enters_probertt == 0) &&
11461                             (bbr->rc_use_google == 0) &&
11462                             bbr->r_ctl.bbr_rttprobe_gain_val &&
11463                             (((cts - bbr->r_ctl.rc_probertt_srttchktim) > bbr_get_rtt(bbr, bbr_drain_rtt)) ||
11464                              (flight >= bbr->r_ctl.flightsize_at_drain))) {
11465                                 /*
11466                                  * We have doddled with our current hptsi
11467                                  * gain an srtt and have still not made it
11468                                  * to target, or we have increased our flight.
11469                                  * Lets reduce the gain by xx%
11470                                  * flooring the reduce at DRAIN (based on
11471                                  * mul/div)
11472                                  */
11473                                 int red;
11474
11475                                 bbr->r_ctl.flightsize_at_drain = flight;
11476                                 bbr->r_ctl.rc_probertt_srttchktim = cts;
11477                                 red = max((bbr->r_ctl.bbr_rttprobe_gain_val / 10), 1);
11478                                 if ((bbr->r_ctl.rc_bbr_hptsi_gain - red) > max(bbr_drain_floor, 1)) {
11479                                         /* Reduce our gain again */
11480                                         bbr->r_ctl.rc_bbr_hptsi_gain -= red;
11481                                         bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG, 0);
11482                                 } else if (bbr->r_ctl.rc_bbr_hptsi_gain > max(bbr_drain_floor, 1)) {
11483                                         /* one more chance before we give up */
11484                                         bbr->r_ctl.rc_bbr_hptsi_gain = max(bbr_drain_floor, 1);
11485                                         bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_SHRINK_PG_FINAL, 0);
11486                                 } else {
11487                                         /* At the very bottom */
11488                                         bbr->r_ctl.rc_bbr_hptsi_gain = max((bbr_drain_floor-1), 1);
11489                                 }
11490                         }
11491                 }
11492                 if (bbr->r_ctl.rc_bbr_enters_probertt &&
11493                     (TSTMP_GT(cts, bbr->r_ctl.rc_bbr_enters_probertt)) &&
11494                     ((cts - bbr->r_ctl.rc_bbr_enters_probertt) >= bbr_rtt_probe_time)) {
11495                         /* Time to exit probe RTT normally */
11496                         bbr_exit_probe_rtt(bbr->rc_tp, bbr, cts);
11497                 }
11498         } else if (bbr->rc_bbr_state == BBR_STATE_PROBE_BW) {
11499                 if ((bbr->rc_tp->snd_una == bbr->rc_tp->snd_max) &&
11500                     (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
11501                         /*
11502                          * This qualifies as a RTT_PROBE session since we
11503                          * drop the data outstanding to nothing and waited
11504                          * more than bbr_rtt_probe_time.
11505                          */
11506                         bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
11507                         bbr_set_reduced_rtt(bbr, cts, __LINE__);
11508                 }
11509                 if (bbr_should_enter_probe_rtt(bbr, cts)) {
11510                         bbr_enter_probe_rtt(bbr, cts, __LINE__);
11511                 } else {
11512                         bbr_set_probebw_gains(bbr, cts, losses);
11513                 }
11514         }
11515 }
11516
11517 static void
11518 bbr_check_bbr_for_state(struct tcp_bbr *bbr, uint32_t cts, int32_t line, uint32_t losses)
11519 {
11520         int32_t epoch = 0;
11521
11522         if ((cts - bbr->r_ctl.rc_rcv_epoch_start) >= bbr_get_rtt(bbr, BBR_RTT_PROP)) {
11523                 bbr_set_epoch(bbr, cts, line);
11524                 /* At each epoch doe lt bw sampling */
11525                 epoch = 1;
11526         }
11527         bbr_state_change(bbr, cts, epoch, bbr->rc_is_pkt_epoch_now, losses);
11528 }
11529
11530 static int
11531 bbr_do_segment_nounlock(struct mbuf *m, struct tcphdr *th, struct socket *so,
11532     struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos,
11533     int32_t nxt_pkt, struct timeval *tv)
11534 {
11535         int32_t thflags, retval;
11536         uint32_t cts, lcts;
11537         uint32_t tiwin;
11538         struct tcpopt to;
11539         struct tcp_bbr *bbr;
11540         struct bbr_sendmap *rsm;
11541         struct timeval ltv;
11542         int32_t did_out = 0;
11543         int32_t in_recovery;
11544         uint16_t nsegs;
11545         int32_t prev_state;
11546         uint32_t lost;
11547
11548         nsegs = max(1, m->m_pkthdr.lro_nsegs);
11549         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
11550         /* add in our stats */
11551         kern_prefetch(bbr, &prev_state);
11552         prev_state = 0;
11553         thflags = th->th_flags;
11554         /*
11555          * If this is either a state-changing packet or current state isn't
11556          * established, we require a write lock on tcbinfo.  Otherwise, we
11557          * allow the tcbinfo to be in either alocked or unlocked, as the
11558          * caller may have unnecessarily acquired a write lock due to a
11559          * race.
11560          */
11561         INP_WLOCK_ASSERT(tp->t_inpcb);
11562         KASSERT(tp->t_state > TCPS_LISTEN, ("%s: TCPS_LISTEN",
11563             __func__));
11564         KASSERT(tp->t_state != TCPS_TIME_WAIT, ("%s: TCPS_TIME_WAIT",
11565             __func__));
11566
11567         tp->t_rcvtime = ticks;
11568         /*
11569          * Unscale the window into a 32-bit value. For the SYN_SENT state
11570          * the scale is zero.
11571          */
11572         tiwin = th->th_win << tp->snd_scale;
11573 #ifdef STATS
11574         stats_voi_update_abs_ulong(tp->t_stats, VOI_TCP_FRWIN, tiwin);
11575 #endif
11576         /*
11577          * Parse options on any incoming segment.
11578          */
11579         tcp_dooptions(&to, (u_char *)(th + 1),
11580             (th->th_off << 2) - sizeof(struct tcphdr),
11581             (thflags & TH_SYN) ? TO_SYN : 0);
11582
11583         if (m->m_flags & M_TSTMP) {
11584                 /* Prefer the hardware timestamp if present */
11585                 struct timespec ts;
11586
11587                 mbuf_tstmp2timespec(m, &ts);
11588                 bbr->rc_tv.tv_sec = ts.tv_sec;
11589                 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11590                 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);
11591         } else if (m->m_flags & M_TSTMP_LRO) {
11592                 /* Next the arrival timestamp */
11593                 struct timespec ts;
11594
11595                 mbuf_tstmp2timespec(m, &ts);
11596                 bbr->rc_tv.tv_sec = ts.tv_sec;
11597                 bbr->rc_tv.tv_usec = ts.tv_nsec / 1000;
11598                 bbr->r_ctl.rc_rcvtime = cts = tcp_tv_to_usectick(&bbr->rc_tv);
11599         } else {
11600                 /*
11601                  * Ok just get the current time.
11602                  */
11603                 bbr->r_ctl.rc_rcvtime = lcts = cts = tcp_get_usecs(&bbr->rc_tv);
11604         }
11605         /*
11606          * If echoed timestamp is later than the current time, fall back to
11607          * non RFC1323 RTT calculation.  Normalize timestamp if syncookies
11608          * were used when this connection was established.
11609          */
11610         if ((to.to_flags & TOF_TS) && (to.to_tsecr != 0)) {
11611                 to.to_tsecr -= tp->ts_offset;
11612                 if (TSTMP_GT(to.to_tsecr, tcp_tv_to_mssectick(&bbr->rc_tv)))
11613                         to.to_tsecr = 0;
11614         }
11615         /*
11616          * If its the first time in we need to take care of options and
11617          * verify we can do SACK for rack!
11618          */
11619         if (bbr->r_state == 0) {
11620                 /*
11621                  * Process options only when we get SYN/ACK back. The SYN
11622                  * case for incoming connections is handled in tcp_syncache.
11623                  * According to RFC1323 the window field in a SYN (i.e., a
11624                  * <SYN> or <SYN,ACK>) segment itself is never scaled. XXX
11625                  * this is traditional behavior, may need to be cleaned up.
11626                  */
11627                 if (bbr->rc_inp == NULL) {
11628                         bbr->rc_inp = tp->t_inpcb;
11629                 }
11630                 /*
11631                  * We need to init rc_inp here since its not init'd when
11632                  * bbr_init is called
11633                  */
11634                 if (tp->t_state == TCPS_SYN_SENT && (thflags & TH_SYN)) {
11635                         if ((to.to_flags & TOF_SCALE) &&
11636                             (tp->t_flags & TF_REQ_SCALE)) {
11637                                 tp->t_flags |= TF_RCVD_SCALE;
11638                                 tp->snd_scale = to.to_wscale;
11639                         }
11640                         /*
11641                          * Initial send window.  It will be updated with the
11642                          * next incoming segment to the scaled value.
11643                          */
11644                         tp->snd_wnd = th->th_win;
11645                         if (to.to_flags & TOF_TS) {
11646                                 tp->t_flags |= TF_RCVD_TSTMP;
11647                                 tp->ts_recent = to.to_tsval;
11648                                 tp->ts_recent_age = tcp_tv_to_mssectick(&bbr->rc_tv);
11649                         }
11650                         if (to.to_flags & TOF_MSS)
11651                                 tcp_mss(tp, to.to_mss);
11652                         if ((tp->t_flags & TF_SACK_PERMIT) &&
11653                             (to.to_flags & TOF_SACKPERM) == 0)
11654                                 tp->t_flags &= ~TF_SACK_PERMIT;
11655                         if (IS_FASTOPEN(tp->t_flags)) {
11656                                 if (to.to_flags & TOF_FASTOPEN) {
11657                                         uint16_t mss;
11658
11659                                         if (to.to_flags & TOF_MSS)
11660                                                 mss = to.to_mss;
11661                                         else
11662                                                 if ((tp->t_inpcb->inp_vflag & INP_IPV6) != 0)
11663                                                         mss = TCP6_MSS;
11664                                                 else
11665                                                         mss = TCP_MSS;
11666                                         tcp_fastopen_update_cache(tp, mss,
11667                                             to.to_tfo_len, to.to_tfo_cookie);
11668                                 } else
11669                                         tcp_fastopen_disable_path(tp);
11670                         }
11671                 }
11672                 /*
11673                  * At this point we are at the initial call. Here we decide
11674                  * if we are doing RACK or not. We do this by seeing if
11675                  * TF_SACK_PERMIT is set, if not rack is *not* possible and
11676                  * we switch to the default code.
11677                  */
11678                 if ((tp->t_flags & TF_SACK_PERMIT) == 0) {
11679                         /* Bail */
11680                         tcp_switch_back_to_default(tp);
11681                         (*tp->t_fb->tfb_tcp_do_segment) (m, th, so, tp, drop_hdrlen,
11682                             tlen, iptos);
11683                         return (1);
11684                 }
11685                 /* Set the flag */
11686                 bbr->r_is_v6 = (tp->t_inpcb->inp_vflag & INP_IPV6) != 0;
11687                 tcp_set_hpts(tp->t_inpcb);
11688                 sack_filter_clear(&bbr->r_ctl.bbr_sf, th->th_ack);
11689         }
11690         if (thflags & TH_ACK) {
11691                 /* Track ack types */
11692                 if (to.to_flags & TOF_SACK)
11693                         BBR_STAT_INC(bbr_acks_with_sacks);
11694                 else
11695                         BBR_STAT_INC(bbr_plain_acks);
11696         }
11697         /*
11698          * This is the one exception case where we set the rack state
11699          * always. All other times (timers etc) we must have a rack-state
11700          * set (so we assure we have done the checks above for SACK).
11701          */
11702         if (bbr->r_state != tp->t_state)
11703                 bbr_set_state(tp, bbr, tiwin);
11704
11705         if (SEQ_GT(th->th_ack, tp->snd_una) && (rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map)) != NULL)
11706                 kern_prefetch(rsm, &prev_state);
11707         prev_state = bbr->r_state;
11708         bbr->rc_ack_was_delayed = 0;
11709         lost = bbr->r_ctl.rc_lost;
11710         bbr->rc_is_pkt_epoch_now = 0;
11711         if (m->m_flags & (M_TSTMP|M_TSTMP_LRO)) {
11712                 /* Get the real time into lcts and figure the real delay */
11713                 lcts = tcp_get_usecs(&ltv);
11714                 if (TSTMP_GT(lcts, cts)) {
11715                         bbr->r_ctl.rc_ack_hdwr_delay = lcts - cts;
11716                         bbr->rc_ack_was_delayed = 1;
11717                         if (TSTMP_GT(bbr->r_ctl.rc_ack_hdwr_delay,
11718                                      bbr->r_ctl.highest_hdwr_delay))
11719                                 bbr->r_ctl.highest_hdwr_delay = bbr->r_ctl.rc_ack_hdwr_delay;
11720                 } else {
11721                         bbr->r_ctl.rc_ack_hdwr_delay = 0;
11722                         bbr->rc_ack_was_delayed = 0;
11723                 }
11724         } else {
11725                 bbr->r_ctl.rc_ack_hdwr_delay = 0;
11726                 bbr->rc_ack_was_delayed = 0;
11727         }
11728         bbr_log_ack_event(bbr, th, &to, tlen, nsegs, cts, nxt_pkt, m);
11729         if ((thflags & TH_SYN) && (thflags & TH_FIN) && V_drop_synfin) {
11730                 retval = 0;
11731                 m_freem(m);
11732                 goto done_with_input;
11733         }
11734         /*
11735          * If a segment with the ACK-bit set arrives in the SYN-SENT state
11736          * check SEQ.ACK first as described on page 66 of RFC 793, section 3.9.
11737          */
11738         if ((tp->t_state == TCPS_SYN_SENT) && (thflags & TH_ACK) &&
11739             (SEQ_LEQ(th->th_ack, tp->iss) || SEQ_GT(th->th_ack, tp->snd_max))) {
11740                 ctf_do_dropwithreset_conn(m, tp, th, BANDLIM_RST_OPENPORT, tlen);
11741                 return (1);
11742         }
11743         in_recovery = IN_RECOVERY(tp->t_flags);
11744         if (tiwin > bbr->r_ctl.rc_high_rwnd)
11745                 bbr->r_ctl.rc_high_rwnd = tiwin;
11746 #ifdef BBR_INVARIANTS
11747         if ((tp->t_inpcb->inp_flags & INP_DROPPED) ||
11748             (tp->t_inpcb->inp_flags2 & INP_FREED)) {
11749                 panic("tp:%p bbr:%p given a dropped inp:%p",
11750                     tp, bbr, tp->t_inpcb);
11751         }
11752 #endif
11753         bbr->r_ctl.rc_flight_at_input = ctf_flight_size(tp,
11754                                             (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11755         bbr->rtt_valid = 0;
11756         if (to.to_flags & TOF_TS) {
11757                 bbr->rc_ts_valid = 1;
11758                 bbr->r_ctl.last_inbound_ts = to.to_tsval;
11759         } else {
11760                 bbr->rc_ts_valid = 0;
11761                 bbr->r_ctl.last_inbound_ts = 0;
11762         }
11763         retval = (*bbr->r_substate) (m, th, so,
11764             tp, &to, drop_hdrlen,
11765             tlen, tiwin, thflags, nxt_pkt);
11766 #ifdef BBR_INVARIANTS
11767         if ((retval == 0) &&
11768             (tp->t_inpcb == NULL)) {
11769                 panic("retval:%d tp:%p t_inpcb:NULL state:%d",
11770                     retval, tp, prev_state);
11771         }
11772 #endif
11773         if (nxt_pkt == 0)
11774                 BBR_STAT_INC(bbr_rlock_left_ret0);
11775         else
11776                 BBR_STAT_INC(bbr_rlock_left_ret1);
11777         if (retval == 0) {
11778                 /*
11779                  * If retval is 1 the tcb is unlocked and most likely the tp
11780                  * is gone.
11781                  */
11782                 INP_WLOCK_ASSERT(tp->t_inpcb);
11783                 tcp_bbr_xmit_timer_commit(bbr, tp, cts);
11784                 if (bbr->rc_is_pkt_epoch_now)
11785                         bbr_set_pktepoch(bbr, cts, __LINE__);
11786                 bbr_check_bbr_for_state(bbr, cts, __LINE__, (bbr->r_ctl.rc_lost - lost));
11787                 if (nxt_pkt == 0) {
11788                         if (bbr->r_wanted_output != 0) {
11789                                 bbr->rc_output_starts_timer = 0;
11790                                 did_out = 1;
11791                                 (void)tp->t_fb->tfb_tcp_output(tp);
11792                         } else
11793                                 bbr_start_hpts_timer(bbr, tp, cts, 6, 0, 0);
11794                 }
11795                 if ((nxt_pkt == 0) &&
11796                     ((bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) == 0) &&
11797                     (SEQ_GT(tp->snd_max, tp->snd_una) ||
11798                      (tp->t_flags & TF_DELACK) ||
11799                      ((V_tcp_always_keepalive || bbr->rc_inp->inp_socket->so_options & SO_KEEPALIVE) &&
11800                       (tp->t_state <= TCPS_CLOSING)))) {
11801                         /*
11802                          * We could not send (probably in the hpts but
11803                          * stopped the timer)?
11804                          */
11805                         if ((tp->snd_max == tp->snd_una) &&
11806                             ((tp->t_flags & TF_DELACK) == 0) &&
11807                             (bbr->rc_inp->inp_in_hpts) &&
11808                             (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
11809                                 /*
11810                                  * keep alive not needed if we are hptsi
11811                                  * output yet
11812                                  */
11813                                 ;
11814                         } else {
11815                                 if (bbr->rc_inp->inp_in_hpts) {
11816                                         tcp_hpts_remove(bbr->rc_inp, HPTS_REMOVE_OUTPUT);
11817                                         if ((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
11818                                             (TSTMP_GT(lcts, bbr->rc_pacer_started))) {
11819                                                 uint32_t del;
11820
11821                                                 del = lcts - bbr->rc_pacer_started;
11822                                                 if (bbr->r_ctl.rc_last_delay_val > del) {
11823                                                         BBR_STAT_INC(bbr_force_timer_start);
11824                                                         bbr->r_ctl.rc_last_delay_val -= del;
11825                                                         bbr->rc_pacer_started = lcts;
11826                                                 } else {
11827                                                         /* We are late */
11828                                                         bbr->r_ctl.rc_last_delay_val = 0;
11829                                                         BBR_STAT_INC(bbr_force_output);
11830                                                         (void)tp->t_fb->tfb_tcp_output(tp);
11831                                                 }
11832                                         }
11833                                 }
11834                                 bbr_start_hpts_timer(bbr, tp, cts, 8, bbr->r_ctl.rc_last_delay_val,
11835                                     0);
11836                         }
11837                 } else if ((bbr->rc_output_starts_timer == 0) && (nxt_pkt == 0)) {
11838                         /* Do we have the correct timer running? */
11839                         bbr_timer_audit(tp, bbr, lcts, &so->so_snd);
11840                 }
11841                 /* Do we have a new state */
11842                 if (bbr->r_state != tp->t_state)
11843                         bbr_set_state(tp, bbr, tiwin);
11844 done_with_input:
11845                 bbr_log_doseg_done(bbr, cts, nxt_pkt, did_out);
11846                 if (did_out)
11847                         bbr->r_wanted_output = 0;
11848 #ifdef BBR_INVARIANTS
11849                 if (tp->t_inpcb == NULL) {
11850                         panic("OP:%d retval:%d tp:%p t_inpcb:NULL state:%d",
11851                             did_out,
11852                             retval, tp, prev_state);
11853                 }
11854 #endif
11855         }
11856         return (retval);
11857 }
11858
11859 static void
11860 bbr_log_type_hrdwtso(struct tcpcb *tp, struct tcp_bbr *bbr, int len, int mod, int what_we_can_send)
11861 {
11862         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
11863                 union tcp_log_stackspecific log;
11864                 struct timeval tv;
11865                 uint32_t cts;
11866
11867                 cts = tcp_get_usecs(&tv);
11868                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
11869                 log.u_bbr.flex1 = bbr->r_ctl.rc_pace_min_segs;
11870                 log.u_bbr.flex2 = what_we_can_send;
11871                 log.u_bbr.flex3 = bbr->r_ctl.rc_pace_max_segs;
11872                 log.u_bbr.flex4 = len;
11873                 log.u_bbr.flex5 = 0;
11874                 log.u_bbr.flex7 = mod;
11875                 log.u_bbr.flex8 = 1;
11876                 TCP_LOG_EVENTP(tp, NULL,
11877                     &tp->t_inpcb->inp_socket->so_rcv,
11878                     &tp->t_inpcb->inp_socket->so_snd,
11879                     TCP_HDWR_TLS, 0,
11880                     0, &log, false, &tv);
11881         }
11882 }
11883
11884 static void
11885 bbr_do_segment(struct mbuf *m, struct tcphdr *th, struct socket *so,
11886     struct tcpcb *tp, int32_t drop_hdrlen, int32_t tlen, uint8_t iptos)
11887 {
11888         struct timeval tv;
11889         int retval;
11890
11891         /* First lets see if we have old packets */
11892         if (tp->t_in_pkt) {
11893                 if (ctf_do_queued_segments(so, tp, 1)) {
11894                         m_freem(m);
11895                         return;
11896                 }
11897         }
11898         if (m->m_flags & M_TSTMP_LRO) {
11899                 tv.tv_sec = m->m_pkthdr.rcv_tstmp /1000000000;
11900                 tv.tv_usec = (m->m_pkthdr.rcv_tstmp % 1000000000)/1000;
11901         } else {
11902                 /* Should not be should we kassert instead? */
11903                 tcp_get_usecs(&tv);
11904         }
11905         retval = bbr_do_segment_nounlock(m, th, so, tp,
11906                                          drop_hdrlen, tlen, iptos, 0, &tv);
11907         if (retval == 0)
11908                 INP_WUNLOCK(tp->t_inpcb);
11909 }
11910
11911 /*
11912  * Return how much data can be sent without violating the
11913  * cwnd or rwnd.
11914  */
11915
11916 static inline uint32_t
11917 bbr_what_can_we_send(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t sendwin,
11918     uint32_t avail, int32_t sb_offset, uint32_t cts)
11919 {
11920         uint32_t len;
11921
11922         if (ctf_outstanding(tp) >= tp->snd_wnd) {
11923                 /* We never want to go over our peers rcv-window */
11924                 len = 0;
11925         } else {
11926                 uint32_t flight;
11927
11928                 flight = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked + bbr->r_ctl.rc_lost_bytes));
11929                 if (flight >= sendwin) {
11930                         /*
11931                          * We have in flight what we are allowed by cwnd (if
11932                          * it was rwnd blocking it would have hit above out
11933                          * >= tp->snd_wnd).
11934                          */
11935                         return (0);
11936                 }
11937                 len = sendwin - flight;
11938                 if ((len + ctf_outstanding(tp)) > tp->snd_wnd) {
11939                         /* We would send too much (beyond the rwnd) */
11940                         len = tp->snd_wnd - ctf_outstanding(tp);
11941                 }
11942                 if ((len + sb_offset) > avail) {
11943                         /*
11944                          * We don't have that much in the SB, how much is
11945                          * there?
11946                          */
11947                         len = avail - sb_offset;
11948                 }
11949         }
11950         return (len);
11951 }
11952
11953 static inline void
11954 bbr_do_error_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error)
11955 {
11956 #ifdef NETFLIX_STATS
11957         KMOD_TCPSTAT_INC(tcps_sndpack_error);
11958         KMOD_TCPSTAT_ADD(tcps_sndbyte_error, len);
11959 #endif
11960 }
11961
11962 static inline void
11963 bbr_do_send_accounting(struct tcpcb *tp, struct tcp_bbr *bbr, struct bbr_sendmap *rsm, int32_t len, int32_t error)
11964 {
11965         if (error) {
11966                 bbr_do_error_accounting(tp, bbr, rsm, len, error);
11967                 return;
11968         }
11969         if ((tp->t_flags & TF_FORCEDATA) && len == 1) {
11970                 /* Window probe */
11971                 KMOD_TCPSTAT_INC(tcps_sndprobe);
11972 #ifdef STATS
11973                 stats_voi_update_abs_u32(tp->t_stats,
11974                     VOI_TCP_RETXPB, len);
11975 #endif
11976         } else if (rsm) {
11977                 if (rsm->r_flags & BBR_TLP) {
11978                         /*
11979                          * TLP should not count in retran count, but in its
11980                          * own bin
11981                          */
11982 #ifdef NETFLIX_STATS
11983                         tp->t_sndtlppack++;
11984                         tp->t_sndtlpbyte += len;
11985                         KMOD_TCPSTAT_INC(tcps_tlpresends);
11986                         KMOD_TCPSTAT_ADD(tcps_tlpresend_bytes, len);
11987 #endif
11988                 } else {
11989                         /* Retransmit */
11990                         tp->t_sndrexmitpack++;
11991                         KMOD_TCPSTAT_INC(tcps_sndrexmitpack);
11992                         KMOD_TCPSTAT_ADD(tcps_sndrexmitbyte, len);
11993 #ifdef STATS
11994                         stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
11995                             len);
11996 #endif
11997                 }
11998                 /*
11999                  * Logs in 0 - 8, 8 is all non probe_bw states 0-7 is
12000                  * sub-state
12001                  */
12002                 counter_u64_add(bbr_state_lost[rsm->r_bbr_state], len);
12003                 if (bbr->rc_bbr_state != BBR_STATE_PROBE_BW) {
12004                         /* Non probe_bw log in 1, 2, or 4. */
12005                         counter_u64_add(bbr_state_resend[bbr->rc_bbr_state], len);
12006                 } else {
12007                         /*
12008                          * Log our probe state 3, and log also 5-13 to show
12009                          * us the recovery sub-state for the send. This
12010                          * means that 3 == (5+6+7+8+9+10+11+12+13)
12011                          */
12012                         counter_u64_add(bbr_state_resend[BBR_STATE_PROBE_BW], len);
12013                         counter_u64_add(bbr_state_resend[(bbr_state_val(bbr) + 5)], len);
12014                 }
12015                 /* Place in both 16's the totals of retransmitted */
12016                 counter_u64_add(bbr_state_lost[16], len);
12017                 counter_u64_add(bbr_state_resend[16], len);
12018                 /* Place in 17's the total sent */
12019                 counter_u64_add(bbr_state_resend[17], len);
12020                 counter_u64_add(bbr_state_lost[17], len);
12021
12022         } else {
12023                 /* New sends */
12024                 KMOD_TCPSTAT_INC(tcps_sndpack);
12025                 KMOD_TCPSTAT_ADD(tcps_sndbyte, len);
12026                 /* Place in 17's the total sent */
12027                 counter_u64_add(bbr_state_resend[17], len);
12028                 counter_u64_add(bbr_state_lost[17], len);
12029 #ifdef STATS
12030                 stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
12031                     len);
12032 #endif
12033         }
12034 }
12035
12036 static void
12037 bbr_cwnd_limiting(struct tcpcb *tp, struct tcp_bbr *bbr, uint32_t in_level)
12038 {
12039         if (bbr->rc_filled_pipe && bbr_target_cwnd_mult_limit && (bbr->rc_use_google == 0)) {
12040                 /*
12041                  * Limit the cwnd to not be above N x the target plus whats
12042                  * is outstanding. The target is based on the current b/w
12043                  * estimate.
12044                  */
12045                 uint32_t target;
12046
12047                 target = bbr_get_target_cwnd(bbr, bbr_get_bw(bbr), BBR_UNIT);
12048                 target += ctf_outstanding(tp);
12049                 target *= bbr_target_cwnd_mult_limit;
12050                 if (tp->snd_cwnd > target)
12051                         tp->snd_cwnd = target;
12052                 bbr_log_type_cwndupd(bbr, 0, 0, 0, 10, 0, 0, __LINE__);
12053         }
12054 }
12055
12056 static int
12057 bbr_window_update_needed(struct tcpcb *tp, struct socket *so, uint32_t recwin, int32_t maxseg)
12058 {
12059         /*
12060          * "adv" is the amount we could increase the window, taking into
12061          * account that we are limited by TCP_MAXWIN << tp->rcv_scale.
12062          */
12063         uint32_t adv;
12064         int32_t oldwin;
12065
12066         adv = min(recwin, TCP_MAXWIN << tp->rcv_scale);
12067         if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
12068                 oldwin = (tp->rcv_adv - tp->rcv_nxt);
12069                 adv -= oldwin;
12070         } else
12071                 oldwin = 0;
12072
12073         /*
12074          * If the new window size ends up being the same as the old size
12075          * when it is scaled, then don't force a window update.
12076          */
12077         if (oldwin >> tp->rcv_scale == (adv + oldwin) >> tp->rcv_scale)
12078                 return (0);
12079
12080         if (adv >= (2 * maxseg) &&
12081             (adv >= (so->so_rcv.sb_hiwat / 4) ||
12082             recwin <= (so->so_rcv.sb_hiwat / 8) ||
12083             so->so_rcv.sb_hiwat <= 8 * maxseg)) {
12084                 return (1);
12085         }
12086         if (2 * adv >= (int32_t) so->so_rcv.sb_hiwat)
12087                 return (1);
12088         return (0);
12089 }
12090
12091 /*
12092  * Return 0 on success and a errno on failure to send.
12093  * Note that a 0 return may not mean we sent anything
12094  * if the TCB was on the hpts. A non-zero return
12095  * does indicate the error we got from ip[6]_output.
12096  */
12097 static int
12098 bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv)
12099 {
12100         struct socket *so;
12101         int32_t len;
12102         uint32_t cts;
12103         uint32_t recwin, sendwin;
12104         int32_t sb_offset;
12105         int32_t flags, abandon, error = 0;
12106         struct tcp_log_buffer *lgb = NULL;
12107         struct mbuf *m;
12108         struct mbuf *mb;
12109         uint32_t if_hw_tsomaxsegcount = 0;
12110         uint32_t if_hw_tsomaxsegsize = 0;
12111         uint32_t if_hw_tsomax = 0;
12112         struct ip *ip = NULL;
12113 #ifdef TCPDEBUG
12114         struct ipovly *ipov = NULL;
12115 #endif
12116         struct tcp_bbr *bbr;
12117         struct tcphdr *th;
12118 #ifdef NETFLIX_TCPOUDP
12119         struct udphdr *udp = NULL;
12120 #endif
12121         u_char opt[TCP_MAXOLEN];
12122         unsigned ipoptlen, optlen, hdrlen;
12123 #ifdef NETFLIX_TCPOUDP
12124         unsigned ulen;
12125 #endif
12126         uint32_t bbr_seq;
12127         uint32_t delay_calc=0;
12128         uint8_t doing_tlp = 0;
12129         uint8_t local_options;
12130 #ifdef BBR_INVARIANTS
12131         uint8_t doing_retran_from = 0;
12132         uint8_t picked_up_retran = 0;
12133 #endif
12134         uint8_t wanted_cookie = 0;
12135         uint8_t more_to_rxt=0;
12136         int32_t prefetch_so_done = 0;
12137         int32_t prefetch_rsm = 0;
12138         uint32_t what_we_can = 0;
12139         uint32_t tot_len = 0;
12140         uint32_t rtr_cnt = 0;
12141         uint32_t maxseg, pace_max_segs, p_maxseg;
12142         int32_t csum_flags;
12143         int32_t hw_tls;
12144 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
12145         unsigned ipsec_optlen = 0;
12146
12147 #endif
12148         volatile int32_t sack_rxmit;
12149         struct bbr_sendmap *rsm = NULL;
12150         int32_t tso, mtu;
12151         int force_tso = 0;
12152         struct tcpopt to;
12153         int32_t slot = 0;
12154         struct inpcb *inp;
12155         struct sockbuf *sb;
12156         uint32_t hpts_calling;
12157 #ifdef INET6
12158         struct ip6_hdr *ip6 = NULL;
12159         int32_t isipv6;
12160 #endif
12161         uint8_t app_limited = BBR_JR_SENT_DATA;
12162         uint8_t filled_all = 0;
12163         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
12164         /* We take a cache hit here */
12165         memcpy(&bbr->rc_tv, tv, sizeof(struct timeval));
12166         cts = tcp_tv_to_usectick(&bbr->rc_tv);
12167         inp = bbr->rc_inp;
12168         so = inp->inp_socket;
12169         sb = &so->so_snd;
12170 #ifdef KERN_TLS
12171         if (sb->sb_flags & SB_TLS_IFNET)
12172                 hw_tls = 1;
12173         else
12174 #endif
12175                 hw_tls = 0;
12176         kern_prefetch(sb, &maxseg);
12177         maxseg = tp->t_maxseg - bbr->rc_last_options;
12178         if (bbr_minseg(bbr) < maxseg) {
12179                 tcp_bbr_tso_size_check(bbr, cts);
12180         }
12181         /* Remove any flags that indicate we are pacing on the inp  */
12182         pace_max_segs = bbr->r_ctl.rc_pace_max_segs;
12183         p_maxseg = min(maxseg, pace_max_segs);
12184         INP_WLOCK_ASSERT(inp);
12185 #ifdef TCP_OFFLOAD
12186         if (tp->t_flags & TF_TOE)
12187                 return (tcp_offload_output(tp));
12188 #endif
12189
12190 #ifdef INET6
12191         if (bbr->r_state) {
12192                 /* Use the cache line loaded if possible */
12193                 isipv6 = bbr->r_is_v6;
12194         } else {
12195                 isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
12196         }
12197 #endif
12198         if (((bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) == 0) &&
12199             inp->inp_in_hpts) {
12200                 /*
12201                  * We are on the hpts for some timer but not hptsi output.
12202                  * Possibly remove from the hpts so we can send/recv etc.
12203                  */
12204                 if ((tp->t_flags & TF_ACKNOW) == 0) {
12205                         /*
12206                          * No immediate demand right now to send an ack, but
12207                          * the user may have read, making room for new data
12208                          * (a window update). If so we may want to cancel
12209                          * whatever timer is running (KEEP/DEL-ACK?) and
12210                          * continue to send out a window update. Or we may
12211                          * have gotten more data into the socket buffer to
12212                          * send.
12213                          */
12214                         recwin = min(max(sbspace(&so->so_rcv), 0),
12215                             TCP_MAXWIN << tp->rcv_scale);
12216                         if ((bbr_window_update_needed(tp, so, recwin, maxseg) == 0) &&
12217                             ((sbavail(sb) + ((tcp_outflags[tp->t_state] & TH_FIN) ? 1 : 0)) <=
12218                             (tp->snd_max - tp->snd_una))) {
12219                                 /*
12220                                  * Nothing new to send and no window update
12221                                  * is needed to send. Lets just return and
12222                                  * let the timer-run off.
12223                                  */
12224                                 return (0);
12225                         }
12226                 }
12227                 tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
12228                 bbr_timer_cancel(bbr, __LINE__, cts);
12229         }
12230         if (bbr->r_ctl.rc_last_delay_val) {
12231                 /* Calculate a rough delay for early escape to sending  */
12232                 if (SEQ_GT(cts, bbr->rc_pacer_started))
12233                         delay_calc = cts - bbr->rc_pacer_started;
12234                 if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
12235                         delay_calc -= bbr->r_ctl.rc_last_delay_val;
12236                 else
12237                         delay_calc = 0;
12238         }
12239         /* Mark that we have called bbr_output(). */
12240         if ((bbr->r_timer_override) ||
12241             (tp->t_flags & TF_FORCEDATA) ||
12242             (tp->t_state < TCPS_ESTABLISHED)) {
12243                 /* Timeouts or early states are exempt */
12244                 if (inp->inp_in_hpts)
12245                         tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
12246         } else if (inp->inp_in_hpts) {
12247                 if ((bbr->r_ctl.rc_last_delay_val) &&
12248                     (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT) &&
12249                     delay_calc) {
12250                         /*
12251                          * We were being paced for output and the delay has
12252                          * already exceeded when we were supposed to be
12253                          * called, lets go ahead and pull out of the hpts
12254                          * and call output.
12255                          */
12256                         counter_u64_add(bbr_out_size[TCP_MSS_ACCT_LATE], 1);
12257                         bbr->r_ctl.rc_last_delay_val = 0;
12258                         tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
12259                 } else if (tp->t_state == TCPS_CLOSED) {
12260                         bbr->r_ctl.rc_last_delay_val = 0;
12261                         tcp_hpts_remove(inp, HPTS_REMOVE_OUTPUT);
12262                 } else {
12263                         /*
12264                          * On the hpts, you shall not pass! even if ACKNOW
12265                          * is on, we will when the hpts fires, unless of
12266                          * course we are overdue.
12267                          */
12268                         counter_u64_add(bbr_out_size[TCP_MSS_ACCT_INPACE], 1);
12269                         return (0);
12270                 }
12271         }
12272         bbr->rc_cwnd_limited = 0;
12273         if (bbr->r_ctl.rc_last_delay_val) {
12274                 /* recalculate the real delay and deal with over/under  */
12275                 if (SEQ_GT(cts, bbr->rc_pacer_started))
12276                         delay_calc = cts - bbr->rc_pacer_started;
12277                 else
12278                         delay_calc = 0;
12279                 if (delay_calc >= bbr->r_ctl.rc_last_delay_val)
12280                         /* Setup the delay which will be added in */
12281                         delay_calc -= bbr->r_ctl.rc_last_delay_val;
12282                 else {
12283                         /*
12284                          * We are early setup to adjust
12285                          * our slot time.
12286                          */
12287                         uint64_t merged_val;
12288
12289                         bbr->r_ctl.rc_agg_early += (bbr->r_ctl.rc_last_delay_val - delay_calc);
12290                         bbr->r_agg_early_set = 1;
12291                         if (bbr->r_ctl.rc_hptsi_agg_delay) {
12292                                 if (bbr->r_ctl.rc_hptsi_agg_delay >= bbr->r_ctl.rc_agg_early) {
12293                                         /* Nope our previous late cancels out the early */
12294                                         bbr->r_ctl.rc_hptsi_agg_delay -= bbr->r_ctl.rc_agg_early;
12295                                         bbr->r_agg_early_set = 0;
12296                                         bbr->r_ctl.rc_agg_early = 0;
12297                                 } else {
12298                                         bbr->r_ctl.rc_agg_early -= bbr->r_ctl.rc_hptsi_agg_delay;
12299                                         bbr->r_ctl.rc_hptsi_agg_delay = 0;
12300                                 }
12301                         }
12302                         merged_val = bbr->rc_pacer_started;
12303                         merged_val <<= 32;
12304                         merged_val |= bbr->r_ctl.rc_last_delay_val;
12305                         bbr_log_pacing_delay_calc(bbr, inp->inp_hpts_calls,
12306                                                  bbr->r_ctl.rc_agg_early, cts, delay_calc, merged_val,
12307                                                  bbr->r_agg_early_set, 3);
12308                         bbr->r_ctl.rc_last_delay_val = 0;
12309                         BBR_STAT_INC(bbr_early);
12310                         delay_calc = 0;
12311                 }
12312         } else {
12313                 /* We were not delayed due to hptsi */
12314                 if (bbr->r_agg_early_set)
12315                         bbr->r_ctl.rc_agg_early = 0;
12316                 bbr->r_agg_early_set = 0;
12317                 delay_calc = 0;
12318         }
12319         if (delay_calc) {
12320                 /*
12321                  * We had a hptsi delay which means we are falling behind on
12322                  * sending at the expected rate. Calculate an extra amount
12323                  * of data we can send, if any, to put us back on track.
12324                  */
12325                 if ((bbr->r_ctl.rc_hptsi_agg_delay + delay_calc) < bbr->r_ctl.rc_hptsi_agg_delay)
12326                         bbr->r_ctl.rc_hptsi_agg_delay = 0xffffffff;
12327                 else
12328                         bbr->r_ctl.rc_hptsi_agg_delay += delay_calc;
12329         }
12330         sendwin = min(tp->snd_wnd, tp->snd_cwnd);
12331         if ((tp->snd_una == tp->snd_max) &&
12332             (bbr->rc_bbr_state != BBR_STATE_IDLE_EXIT) &&
12333             (sbavail(sb))) {
12334                 /*
12335                  * Ok we have been idle with nothing outstanding
12336                  * we possibly need to start fresh with either a new
12337                  * suite of states or a fast-ramp up.
12338                  */
12339                 bbr_restart_after_idle(bbr,
12340                                        cts, bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time));
12341         }
12342         /*
12343          * Now was there a hptsi delay where we are behind? We only count
12344          * being behind if: a) We are not in recovery. b) There was a delay.
12345          * <and> c) We had room to send something.
12346          *
12347          */
12348         hpts_calling = inp->inp_hpts_calls;
12349         inp->inp_hpts_calls = 0;
12350         if (bbr->r_ctl.rc_hpts_flags & PACE_TMR_MASK) {
12351                 if (bbr_process_timers(tp, bbr, cts, hpts_calling)) {
12352                         counter_u64_add(bbr_out_size[TCP_MSS_ACCT_ATIMER], 1);
12353                         return (0);
12354                 }
12355         }
12356         bbr->rc_inp->inp_flags2 &= ~INP_MBUF_QUEUE_READY;
12357         if (hpts_calling &&
12358             (bbr->r_ctl.rc_hpts_flags & PACE_PKT_OUTPUT)) {
12359                 bbr->r_ctl.rc_last_delay_val = 0;
12360         }
12361         bbr->r_timer_override = 0;
12362         bbr->r_wanted_output = 0;
12363         /*
12364          * For TFO connections in SYN_RECEIVED, only allow the initial
12365          * SYN|ACK and those sent by the retransmit timer.
12366          */
12367         if (IS_FASTOPEN(tp->t_flags) &&
12368             ((tp->t_state == TCPS_SYN_RECEIVED) ||
12369              (tp->t_state == TCPS_SYN_SENT)) &&
12370             SEQ_GT(tp->snd_max, tp->snd_una) && /* initial SYN or SYN|ACK sent */
12371             (tp->t_rxtshift == 0)) {    /* not a retransmit */
12372                 return (0);
12373         }
12374         /*
12375          * Before sending anything check for a state update. For hpts
12376          * calling without input this is important. If its input calling
12377          * then this was already done.
12378          */
12379         if (bbr->rc_use_google == 0)
12380                 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
12381 again:
12382         /*
12383          * If we've recently taken a timeout, snd_max will be greater than
12384          * snd_max. BBR in general does not pay much attention to snd_nxt
12385          * for historic reasons the persist timer still uses it. This means
12386          * we have to look at it. All retransmissions that are not persits
12387          * use the rsm that needs to be sent so snd_nxt is ignored. At the
12388          * end of this routine we pull snd_nxt always up to snd_max.
12389          */
12390         doing_tlp = 0;
12391 #ifdef BBR_INVARIANTS
12392         doing_retran_from = picked_up_retran = 0;
12393 #endif
12394         error = 0;
12395         tso = 0;
12396         slot = 0;
12397         mtu = 0;
12398         sendwin = min(tp->snd_wnd, tp->snd_cwnd);
12399         sb_offset = tp->snd_max - tp->snd_una;
12400         flags = tcp_outflags[tp->t_state];
12401         sack_rxmit = 0;
12402         len = 0;
12403         rsm = NULL;
12404         if (flags & TH_RST) {
12405                 SOCKBUF_LOCK(sb);
12406                 goto send;
12407         }
12408 recheck_resend:
12409         while (bbr->r_ctl.rc_free_cnt < bbr_min_req_free) {
12410                 /* We need to always have one in reserve */
12411                 rsm = bbr_alloc(bbr);
12412                 if (rsm == NULL) {
12413                         error = ENOMEM;
12414                         /* Lie to get on the hpts */
12415                         tot_len = tp->t_maxseg;
12416                         if (hpts_calling)
12417                                 /* Retry in a ms */
12418                                 slot = 1001;
12419                         goto just_return_nolock;
12420                 }
12421                 TAILQ_INSERT_TAIL(&bbr->r_ctl.rc_free, rsm, r_next);
12422                 bbr->r_ctl.rc_free_cnt++;
12423                 rsm = NULL;
12424         }
12425         /* What do we send, a resend? */
12426         if (bbr->r_ctl.rc_resend == NULL) {
12427                 /* Check for rack timeout */
12428                 bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts);
12429                 if (bbr->r_ctl.rc_resend) {
12430 #ifdef BBR_INVARIANTS
12431                         picked_up_retran = 1;
12432 #endif
12433                         bbr_cong_signal(tp, NULL, CC_NDUPACK, bbr->r_ctl.rc_resend);
12434                 }
12435         }
12436         if (bbr->r_ctl.rc_resend) {
12437                 rsm = bbr->r_ctl.rc_resend;
12438 #ifdef BBR_INVARIANTS
12439                 doing_retran_from = 1;
12440 #endif
12441                 /* Remove any TLP flags its a RACK or T-O */
12442                 rsm->r_flags &= ~BBR_TLP;
12443                 bbr->r_ctl.rc_resend = NULL;
12444                 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
12445 #ifdef BBR_INVARIANTS
12446                         panic("Huh, tp:%p bbr:%p rsm:%p start:%u < snd_una:%u\n",
12447                             tp, bbr, rsm, rsm->r_start, tp->snd_una);
12448                         goto recheck_resend;
12449 #else
12450                         /* TSNH */
12451                         rsm = NULL;
12452                         goto recheck_resend;
12453 #endif
12454                 }
12455                 rtr_cnt++;
12456                 if (rsm->r_flags & BBR_HAS_SYN) {
12457                         /* Only retransmit a SYN by itself */
12458                         len = 0;
12459                         if ((flags & TH_SYN) == 0) {
12460                                 /* Huh something is wrong */
12461                                 rsm->r_start++;
12462                                 if (rsm->r_start == rsm->r_end) {
12463                                         /* Clean it up, somehow we missed the ack? */
12464                                         bbr_log_syn(tp, NULL);
12465                                 } else {
12466                                         /* TFO with data? */
12467                                         rsm->r_flags &= ~BBR_HAS_SYN;
12468                                         len = rsm->r_end - rsm->r_start;
12469                                 }
12470                         } else {
12471                                 /* Retransmitting SYN */
12472                                 rsm = NULL;
12473                                 SOCKBUF_LOCK(sb);
12474                                 goto send;
12475                         }
12476                 } else
12477                         len = rsm->r_end - rsm->r_start;
12478                 if ((bbr->rc_resends_use_tso == 0) &&
12479 #ifdef KERN_TLS
12480                     ((sb->sb_flags & SB_TLS_IFNET) == 0) &&
12481 #endif
12482                     (len > maxseg)) {
12483                         len = maxseg;
12484                         more_to_rxt = 1;
12485                 }
12486                 sb_offset = rsm->r_start - tp->snd_una;
12487                 if (len > 0) {
12488                         sack_rxmit = 1;
12489                         KMOD_TCPSTAT_INC(tcps_sack_rexmits);
12490                         KMOD_TCPSTAT_ADD(tcps_sack_rexmit_bytes,
12491                             min(len, maxseg));
12492                 } else {
12493                         /* I dont think this can happen */
12494                         rsm = NULL;
12495                         goto recheck_resend;
12496                 }
12497                 BBR_STAT_INC(bbr_resends_set);
12498         } else if (bbr->r_ctl.rc_tlp_send) {
12499                 /*
12500                  * Tail loss probe
12501                  */
12502                 doing_tlp = 1;
12503                 rsm = bbr->r_ctl.rc_tlp_send;
12504                 bbr->r_ctl.rc_tlp_send = NULL;
12505                 sack_rxmit = 1;
12506                 len = rsm->r_end - rsm->r_start;
12507                 rtr_cnt++;
12508                 if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12509                         len = maxseg;
12510
12511                 if (SEQ_GT(tp->snd_una, rsm->r_start)) {
12512 #ifdef BBR_INVARIANTS
12513                         panic("tp:%p bbc:%p snd_una:%u rsm:%p r_start:%u",
12514                             tp, bbr, tp->snd_una, rsm, rsm->r_start);
12515 #else
12516                         /* TSNH */
12517                         rsm = NULL;
12518                         goto recheck_resend;
12519 #endif
12520                 }
12521                 sb_offset = rsm->r_start - tp->snd_una;
12522                 BBR_STAT_INC(bbr_tlp_set);
12523         }
12524         /*
12525          * Enforce a connection sendmap count limit if set
12526          * as long as we are not retransmiting.
12527          */
12528         if ((rsm == NULL) &&
12529             (V_tcp_map_entries_limit > 0) &&
12530             (bbr->r_ctl.rc_num_maps_alloced >= V_tcp_map_entries_limit)) {
12531                 BBR_STAT_INC(bbr_alloc_limited);
12532                 if (!bbr->alloc_limit_reported) {
12533                         bbr->alloc_limit_reported = 1;
12534                         BBR_STAT_INC(bbr_alloc_limited_conns);
12535                 }
12536                 goto just_return_nolock;
12537         }
12538 #ifdef BBR_INVARIANTS
12539         if (rsm && SEQ_LT(rsm->r_start, tp->snd_una)) {
12540                 panic("tp:%p bbr:%p rsm:%p sb_offset:%u len:%u",
12541                     tp, bbr, rsm, sb_offset, len);
12542         }
12543 #endif
12544         /*
12545          * Get standard flags, and add SYN or FIN if requested by 'hidden'
12546          * state flags.
12547          */
12548         if (tp->t_flags & TF_NEEDFIN && (rsm == NULL))
12549                 flags |= TH_FIN;
12550         if (tp->t_flags & TF_NEEDSYN)
12551                 flags |= TH_SYN;
12552
12553         if (rsm && (rsm->r_flags & BBR_HAS_FIN)) {
12554                 /* we are retransmitting the fin */
12555                 len--;
12556                 if (len) {
12557                         /*
12558                          * When retransmitting data do *not* include the
12559                          * FIN. This could happen from a TLP probe if we
12560                          * allowed data with a FIN.
12561                          */
12562                         flags &= ~TH_FIN;
12563                 }
12564         } else if (rsm) {
12565                 if (flags & TH_FIN)
12566                         flags &= ~TH_FIN;
12567         }
12568         if ((sack_rxmit == 0) && (prefetch_rsm == 0)) {
12569                 void *end_rsm;
12570
12571                 end_rsm = TAILQ_LAST_FAST(&bbr->r_ctl.rc_tmap, bbr_sendmap, r_tnext);
12572                 if (end_rsm)
12573                         kern_prefetch(end_rsm, &prefetch_rsm);
12574                 prefetch_rsm = 1;
12575         }
12576         SOCKBUF_LOCK(sb);
12577         /*
12578          * If in persist timeout with window of 0, send 1 byte. Otherwise,
12579          * if window is small but nonzero and time TF_SENTFIN expired, we
12580          * will send what we can and go to transmit state.
12581          */
12582         if (tp->t_flags & TF_FORCEDATA) {
12583                 if ((sendwin == 0) || (sendwin <= (tp->snd_max - tp->snd_una))) {
12584                         /*
12585                          * If we still have some data to send, then clear
12586                          * the FIN bit.  Usually this would happen below
12587                          * when it realizes that we aren't sending all the
12588                          * data.  However, if we have exactly 1 byte of
12589                          * unsent data, then it won't clear the FIN bit
12590                          * below, and if we are in persist state, we wind up
12591                          * sending the packet without recording that we sent
12592                          * the FIN bit.
12593                          *
12594                          * We can't just blindly clear the FIN bit, because
12595                          * if we don't have any more data to send then the
12596                          * probe will be the FIN itself.
12597                          */
12598                         if (sb_offset < sbused(sb))
12599                                 flags &= ~TH_FIN;
12600                         sendwin = 1;
12601                 } else {
12602                         if ((bbr->rc_in_persist != 0) &&
12603                             (tp->snd_wnd >= min((bbr->r_ctl.rc_high_rwnd/2),
12604                                                bbr_minseg(bbr)))) {
12605                                 /* Exit persists if there is space */
12606                                 bbr_exit_persist(tp, bbr, cts, __LINE__);
12607                         }
12608                         if (rsm == NULL) {
12609                                 /*
12610                                  * If we are dropping persist mode then we
12611                                  * need to correct sb_offset if not a
12612                                  * retransmit.
12613                                  */
12614                                 sb_offset = tp->snd_max - tp->snd_una;
12615                         }
12616                 }
12617         }
12618         /*
12619          * If snd_nxt == snd_max and we have transmitted a FIN, the
12620          * sb_offset will be > 0 even if so_snd.sb_cc is 0, resulting in a
12621          * negative length.  This can also occur when TCP opens up its
12622          * congestion window while receiving additional duplicate acks after
12623          * fast-retransmit because TCP will reset snd_nxt to snd_max after
12624          * the fast-retransmit.
12625          *
12626          * In the normal retransmit-FIN-only case, however, snd_nxt will be
12627          * set to snd_una, the sb_offset will be 0, and the length may wind
12628          * up 0.
12629          *
12630          * If sack_rxmit is true we are retransmitting from the scoreboard
12631          * in which case len is already set.
12632          */
12633         if (sack_rxmit == 0) {
12634                 uint32_t avail;
12635
12636                 avail = sbavail(sb);
12637                 if (SEQ_GT(tp->snd_max, tp->snd_una))
12638                         sb_offset = tp->snd_max - tp->snd_una;
12639                 else
12640                         sb_offset = 0;
12641                 if (bbr->rc_tlp_new_data) {
12642                         /* TLP is forcing out new data */
12643                         uint32_t tlplen;
12644
12645                         doing_tlp = 1;
12646                         tlplen = maxseg;
12647
12648                         if (tlplen > (uint32_t)(avail - sb_offset)) {
12649                                 tlplen = (uint32_t)(avail - sb_offset);
12650                         }
12651                         if (tlplen > tp->snd_wnd) {
12652                                 len = tp->snd_wnd;
12653                         } else {
12654                                 len = tlplen;
12655                         }
12656                         bbr->rc_tlp_new_data = 0;
12657                 } else {
12658                         what_we_can = len = bbr_what_can_we_send(tp, bbr, sendwin, avail, sb_offset, cts);
12659                         if ((len < p_maxseg) &&
12660                             (bbr->rc_in_persist == 0) &&
12661                             (ctf_outstanding(tp) >= (2 * p_maxseg)) &&
12662                             ((avail - sb_offset) >= p_maxseg)) {
12663                                 /*
12664                                  * We are not completing whats in the socket
12665                                  * buffer (i.e. there is at least a segment
12666                                  * waiting to send) and we have 2 or more
12667                                  * segments outstanding. There is no sense
12668                                  * of sending a little piece. Lets defer and
12669                                  * and wait until we can send a whole
12670                                  * segment.
12671                                  */
12672                                 len = 0;
12673                         }
12674                         if ((tp->t_flags & TF_FORCEDATA) && (bbr->rc_in_persist)) {
12675                                 /*
12676                                  * We are in persists, figure out if
12677                                  * a retransmit is available (maybe the previous
12678                                  * persists we sent) or if we have to send new
12679                                  * data.
12680                                  */
12681                                 rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
12682                                 if (rsm) {
12683                                         len = rsm->r_end - rsm->r_start;
12684                                         if (rsm->r_flags & BBR_HAS_FIN)
12685                                                 len--;
12686                                         if ((bbr->rc_resends_use_tso == 0) && (len > maxseg))
12687                                                 len = maxseg;
12688                                         if (len > 1)
12689                                                 BBR_STAT_INC(bbr_persist_reneg);
12690                                         /*
12691                                          * XXXrrs we could force the len to
12692                                          * 1 byte here to cause the chunk to
12693                                          * split apart.. but that would then
12694                                          * mean we always retransmit it as
12695                                          * one byte even after the window
12696                                          * opens.
12697                                          */
12698                                         sack_rxmit = 1;
12699                                         sb_offset = rsm->r_start - tp->snd_una;
12700                                 } else {
12701                                         /*
12702                                          * First time through in persists or peer
12703                                          * acked our one byte. Though we do have
12704                                          * to have something in the sb.
12705                                          */
12706                                         len = 1;
12707                                         sb_offset = 0;
12708                                         if (avail == 0)
12709                                             len = 0;
12710                                 }
12711                         }
12712                 }
12713         }
12714         if (prefetch_so_done == 0) {
12715                 kern_prefetch(so, &prefetch_so_done);
12716                 prefetch_so_done = 1;
12717         }
12718         /*
12719          * Lop off SYN bit if it has already been sent.  However, if this is
12720          * SYN-SENT state and if segment contains data and if we don't know
12721          * that foreign host supports TAO, suppress sending segment.
12722          */
12723         if ((flags & TH_SYN) && (rsm == NULL) &&
12724             SEQ_GT(tp->snd_max, tp->snd_una)) {
12725                 if (tp->t_state != TCPS_SYN_RECEIVED)
12726                         flags &= ~TH_SYN;
12727                 /*
12728                  * When sending additional segments following a TFO SYN|ACK,
12729                  * do not include the SYN bit.
12730                  */
12731                 if (IS_FASTOPEN(tp->t_flags) &&
12732                     (tp->t_state == TCPS_SYN_RECEIVED))
12733                         flags &= ~TH_SYN;
12734                 sb_offset--, len++;
12735                 if (sbavail(sb) == 0)
12736                         len = 0;
12737         } else if ((flags & TH_SYN) && rsm) {
12738                 /*
12739                  * Subtract one from the len for the SYN being
12740                  * retransmitted.
12741                  */
12742                 len--;
12743         }
12744         /*
12745          * Be careful not to send data and/or FIN on SYN segments. This
12746          * measure is needed to prevent interoperability problems with not
12747          * fully conformant TCP implementations.
12748          */
12749         if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
12750                 len = 0;
12751                 flags &= ~TH_FIN;
12752         }
12753         /*
12754          * On TFO sockets, ensure no data is sent in the following cases:
12755          *
12756          *  - When retransmitting SYN|ACK on a passively-created socket
12757          *  - When retransmitting SYN on an actively created socket
12758          *  - When sending a zero-length cookie (cookie request) on an
12759          *    actively created socket
12760          *  - When the socket is in the CLOSED state (RST is being sent)
12761          */
12762         if (IS_FASTOPEN(tp->t_flags) &&
12763             (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||
12764              ((tp->t_state == TCPS_SYN_SENT) &&
12765               (tp->t_tfo_client_cookie_len == 0)) ||
12766              (flags & TH_RST))) {
12767                 len = 0;
12768                 sack_rxmit = 0;
12769                 rsm = NULL;
12770         }
12771         /* Without fast-open there should never be data sent on a SYN */
12772         if ((flags & TH_SYN) && (!IS_FASTOPEN(tp->t_flags)))
12773                 len = 0;
12774         if (len <= 0) {
12775                 /*
12776                  * If FIN has been sent but not acked, but we haven't been
12777                  * called to retransmit, len will be < 0.  Otherwise, window
12778                  * shrank after we sent into it.  If window shrank to 0,
12779                  * cancel pending retransmit, pull snd_nxt back to (closed)
12780                  * window, and set the persist timer if it isn't already
12781                  * going.  If the window didn't close completely, just wait
12782                  * for an ACK.
12783                  *
12784                  * We also do a general check here to ensure that we will
12785                  * set the persist timer when we have data to send, but a
12786                  * 0-byte window. This makes sure the persist timer is set
12787                  * even if the packet hits one of the "goto send" lines
12788                  * below.
12789                  */
12790                 len = 0;
12791                 if ((tp->snd_wnd == 0) &&
12792                     (TCPS_HAVEESTABLISHED(tp->t_state)) &&
12793                     (tp->snd_una == tp->snd_max) &&
12794                     (sb_offset < (int)sbavail(sb))) {
12795                         /*
12796                          * Not enough room in the rwnd to send
12797                          * a paced segment out.
12798                          */
12799                         bbr_enter_persist(tp, bbr, cts, __LINE__);
12800                 }
12801         } else if ((rsm == NULL) &&
12802                    (doing_tlp == 0) &&
12803                    (len < bbr->r_ctl.rc_pace_max_segs)) {
12804                 /*
12805                  * We are not sending a full segment for
12806                  * some reason. Should we not send anything (think
12807                  * sws or persists)?
12808                  */
12809                 if ((tp->snd_wnd < min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12810                     (TCPS_HAVEESTABLISHED(tp->t_state)) &&
12811                     (len < (int)(sbavail(sb) - sb_offset))) {
12812                         /*
12813                          * Here the rwnd is less than
12814                          * the pacing size, this is not a retransmit,
12815                          * we are established and
12816                          * the send is not the last in the socket buffer
12817                          * lets not send, and possibly enter persists.
12818                          */
12819                         len = 0;
12820                         if (tp->snd_max == tp->snd_una)
12821                                 bbr_enter_persist(tp, bbr, cts, __LINE__);
12822                 } else if ((tp->snd_cwnd >= bbr->r_ctl.rc_pace_max_segs) &&
12823                            (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12824                                                  bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12825                            (len < (int)(sbavail(sb) - sb_offset)) &&
12826                            (len < bbr_minseg(bbr))) {
12827                         /*
12828                          * Here we are not retransmitting, and
12829                          * the cwnd is not so small that we could
12830                          * not send at least a min size (rxt timer
12831                          * not having gone off), We have 2 segments or
12832                          * more already in flight, its not the tail end
12833                          * of the socket buffer  and the cwnd is blocking
12834                          * us from sending out minimum pacing segment size.
12835                          * Lets not send anything.
12836                          */
12837                         bbr->rc_cwnd_limited = 1;
12838                         len = 0;
12839                 } else if (((tp->snd_wnd - ctf_outstanding(tp)) <
12840                             min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) &&
12841                            (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
12842                                                  bbr->r_ctl.rc_lost_bytes)) > (2 * maxseg)) &&
12843                            (len < (int)(sbavail(sb) - sb_offset)) &&
12844                            (TCPS_HAVEESTABLISHED(tp->t_state))) {
12845                         /*
12846                          * Here we have a send window but we have
12847                          * filled it up and we can't send another pacing segment.
12848                          * We also have in flight more than 2 segments
12849                          * and we are not completing the sb i.e. we allow
12850                          * the last bytes of the sb to go out even if
12851                          * its not a full pacing segment.
12852                          */
12853                         len = 0;
12854                 }
12855         }
12856         /* len will be >= 0 after this point. */
12857         KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
12858         tcp_sndbuf_autoscale(tp, so, sendwin);
12859         /*
12860          *
12861          */
12862         if (bbr->rc_in_persist &&
12863             len &&
12864             (rsm == NULL) &&
12865             (len < min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs))) {
12866                 /*
12867                  * We are in persist, not doing a retransmit and don't have enough space
12868                  * yet to send a full TSO. So is it at the end of the sb
12869                  * if so we need to send else nuke to 0 and don't send.
12870                  */
12871                 int sbleft;
12872                 if (sbavail(sb) > sb_offset)
12873                         sbleft = sbavail(sb) - sb_offset;
12874                 else
12875                         sbleft = 0;
12876                 if (sbleft >= min((bbr->r_ctl.rc_high_rwnd/2), bbr->r_ctl.rc_pace_max_segs)) {
12877                         /* not at end of sb lets not send */
12878                         len = 0;
12879                 }
12880         }
12881         /*
12882          * Decide if we can use TCP Segmentation Offloading (if supported by
12883          * hardware).
12884          *
12885          * TSO may only be used if we are in a pure bulk sending state.  The
12886          * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP
12887          * options prevent using TSO.  With TSO the TCP header is the same
12888          * (except for the sequence number) for all generated packets.  This
12889          * makes it impossible to transmit any options which vary per
12890          * generated segment or packet.
12891          *
12892          * IPv4 handling has a clear separation of ip options and ip header
12893          * flags while IPv6 combines both in in6p_outputopts. ip6_optlen()
12894          * does the right thing below to provide length of just ip options
12895          * and thus checking for ipoptlen is enough to decide if ip options
12896          * are present.
12897          */
12898 #ifdef INET6
12899         if (isipv6)
12900                 ipoptlen = ip6_optlen(inp);
12901         else
12902 #endif
12903         if (inp->inp_options)
12904                 ipoptlen = inp->inp_options->m_len -
12905                     offsetof(struct ipoption, ipopt_list);
12906         else
12907                 ipoptlen = 0;
12908 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
12909         /*
12910          * Pre-calculate here as we save another lookup into the darknesses
12911          * of IPsec that way and can actually decide if TSO is ok.
12912          */
12913 #ifdef INET6
12914         if (isipv6 && IPSEC_ENABLED(ipv6))
12915                 ipsec_optlen = IPSEC_HDRSIZE(ipv6, inp);
12916 #ifdef INET
12917         else
12918 #endif
12919 #endif                          /* INET6 */
12920 #ifdef INET
12921         if (IPSEC_ENABLED(ipv4))
12922                 ipsec_optlen = IPSEC_HDRSIZE(ipv4, inp);
12923 #endif                          /* INET */
12924 #endif                          /* IPSEC */
12925 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
12926         ipoptlen += ipsec_optlen;
12927 #endif
12928         if ((tp->t_flags & TF_TSO) && V_tcp_do_tso &&
12929             (len > maxseg) &&
12930             (tp->t_port == 0) &&
12931             ((tp->t_flags & TF_SIGNATURE) == 0) &&
12932             tp->rcv_numsacks == 0 &&
12933             ipoptlen == 0)
12934                 tso = 1;
12935
12936         recwin = min(max(sbspace(&so->so_rcv), 0),
12937             TCP_MAXWIN << tp->rcv_scale);
12938         /*
12939          * Sender silly window avoidance.   We transmit under the following
12940          * conditions when len is non-zero:
12941          *
12942          * - We have a full segment (or more with TSO) - This is the last
12943          * buffer in a write()/send() and we are either idle or running
12944          * NODELAY - we've timed out (e.g. persist timer) - we have more
12945          * then 1/2 the maximum send window's worth of data (receiver may be
12946          * limited the window size) - we need to retransmit
12947          */
12948         if (rsm)
12949                 goto send;
12950         if (len) {
12951                 if (sack_rxmit)
12952                         goto send;
12953                 if (len >= p_maxseg)
12954                         goto send;
12955                 /*
12956                  * NOTE! on localhost connections an 'ack' from the remote
12957                  * end may occur synchronously with the output and cause us
12958                  * to flush a buffer queued with moretocome.  XXX
12959                  *
12960                  */
12961                 if (((tp->t_flags & TF_MORETOCOME) == 0) &&     /* normal case */
12962                     ((tp->t_flags & TF_NODELAY) ||
12963                     ((uint32_t)len + (uint32_t)sb_offset) >= sbavail(&so->so_snd)) &&
12964                     (tp->t_flags & TF_NOPUSH) == 0) {
12965                         goto send;
12966                 }
12967                 if ((tp->snd_una == tp->snd_max) && len) {      /* Nothing outstanding */
12968                         goto send;
12969                 }
12970                 if (tp->t_flags & TF_FORCEDATA) {       /* typ. timeout case */
12971                         goto send;
12972                 }
12973                 if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0) {
12974                         goto send;
12975                 }
12976         }
12977         /*
12978          * Sending of standalone window updates.
12979          *
12980          * Window updates are important when we close our window due to a
12981          * full socket buffer and are opening it again after the application
12982          * reads data from it.  Once the window has opened again and the
12983          * remote end starts to send again the ACK clock takes over and
12984          * provides the most current window information.
12985          *
12986          * We must avoid the silly window syndrome whereas every read from
12987          * the receive buffer, no matter how small, causes a window update
12988          * to be sent.  We also should avoid sending a flurry of window
12989          * updates when the socket buffer had queued a lot of data and the
12990          * application is doing small reads.
12991          *
12992          * Prevent a flurry of pointless window updates by only sending an
12993          * update when we can increase the advertized window by more than
12994          * 1/4th of the socket buffer capacity.  When the buffer is getting
12995          * full or is very small be more aggressive and send an update
12996          * whenever we can increase by two mss sized segments. In all other
12997          * situations the ACK's to new incoming data will carry further
12998          * window increases.
12999          *
13000          * Don't send an independent window update if a delayed ACK is
13001          * pending (it will get piggy-backed on it) or the remote side
13002          * already has done a half-close and won't send more data.  Skip
13003          * this if the connection is in T/TCP half-open state.
13004          */
13005         if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&
13006             !(tp->t_flags & TF_DELACK) &&
13007             !TCPS_HAVERCVDFIN(tp->t_state)) {
13008                 /* Check to see if we should do a window update */
13009                 if (bbr_window_update_needed(tp, so, recwin, maxseg))
13010                         goto send;
13011         }
13012         /*
13013          * Send if we owe the peer an ACK, RST, SYN, or urgent data.  ACKNOW
13014          * is also a catch-all for the retransmit timer timeout case.
13015          */
13016         if (tp->t_flags & TF_ACKNOW) {
13017                 goto send;
13018         }
13019         if (((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0)) {
13020                 goto send;
13021         }
13022         if (SEQ_GT(tp->snd_up, tp->snd_una)) {
13023                 goto send;
13024         }
13025         /*
13026          * If our state indicates that FIN should be sent and we have not
13027          * yet done so, then we need to send.
13028          */
13029         if (flags & TH_FIN &&
13030             ((tp->t_flags & TF_SENTFIN) == 0)) {
13031                 goto send;
13032         }
13033         /*
13034          * No reason to send a segment, just return.
13035          */
13036 just_return:
13037         SOCKBUF_UNLOCK(sb);
13038 just_return_nolock:
13039         if (tot_len)
13040                 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
13041         if (bbr->rc_no_pacing)
13042                 slot = 0;
13043         if (tot_len == 0) {
13044                 if ((ctf_outstanding(tp) + min((bbr->r_ctl.rc_high_rwnd/2), bbr_minseg(bbr))) >=
13045                     tp->snd_wnd) {
13046                         BBR_STAT_INC(bbr_rwnd_limited);
13047                         app_limited = BBR_JR_RWND_LIMITED;
13048                         bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
13049                         if ((bbr->rc_in_persist == 0) &&
13050                             TCPS_HAVEESTABLISHED(tp->t_state) &&
13051                             (tp->snd_max == tp->snd_una) &&
13052                             sbavail(&tp->t_inpcb->inp_socket->so_snd)) {
13053                                 /* No send window.. we must enter persist */
13054                                 bbr_enter_persist(tp, bbr, bbr->r_ctl.rc_rcvtime, __LINE__);
13055                         }
13056                 } else if (ctf_outstanding(tp) >= sbavail(sb)) {
13057                         BBR_STAT_INC(bbr_app_limited);
13058                         app_limited = BBR_JR_APP_LIMITED;
13059                         bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
13060                 } else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
13061                                                  bbr->r_ctl.rc_lost_bytes)) + p_maxseg) >= tp->snd_cwnd) {
13062                         BBR_STAT_INC(bbr_cwnd_limited);
13063                         app_limited = BBR_JR_CWND_LIMITED;
13064                         bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
13065                                                                         bbr->r_ctl.rc_lost_bytes)));
13066                         bbr->rc_cwnd_limited = 1;
13067                 } else {
13068                         BBR_STAT_INC(bbr_app_limited);
13069                         app_limited = BBR_JR_APP_LIMITED;
13070                         bbr_cwnd_limiting(tp, bbr, ctf_outstanding(tp));
13071                 }
13072                 bbr->r_ctl.rc_hptsi_agg_delay = 0;
13073                 bbr->r_agg_early_set = 0;
13074                 bbr->r_ctl.rc_agg_early = 0;
13075                 bbr->r_ctl.rc_last_delay_val = 0;
13076         } else if (bbr->rc_use_google == 0)
13077                 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
13078         /* Are we app limited? */
13079         if ((app_limited == BBR_JR_APP_LIMITED) ||
13080             (app_limited == BBR_JR_RWND_LIMITED)) {
13081                 /**
13082                  * We are application limited.
13083                  */
13084                 bbr->r_ctl.r_app_limited_until = (ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
13085                                                                        bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_delivered);
13086         }
13087         if (tot_len == 0)
13088                 counter_u64_add(bbr_out_size[TCP_MSS_ACCT_JUSTRET], 1);
13089         tp->t_flags &= ~TF_FORCEDATA;
13090         /* Dont update the time if we did not send */
13091         bbr->r_ctl.rc_last_delay_val = 0;
13092         bbr->rc_output_starts_timer = 1;
13093         bbr_start_hpts_timer(bbr, tp, cts, 9, slot, tot_len);
13094         bbr_log_type_just_return(bbr, cts, tot_len, hpts_calling, app_limited, p_maxseg, len);
13095         if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
13096                 /* Make sure snd_nxt is drug up */
13097                 tp->snd_nxt = tp->snd_max;
13098         }
13099         return (error);
13100
13101 send:
13102         if (doing_tlp == 0) {
13103                 /*
13104                  * Data not a TLP, and its not the rxt firing. If it is the
13105                  * rxt firing, we want to leave the tlp_in_progress flag on
13106                  * so we don't send another TLP. It has to be a rack timer
13107                  * or normal send (response to acked data) to clear the tlp
13108                  * in progress flag.
13109                  */
13110                 bbr->rc_tlp_in_progress = 0;
13111                 bbr->rc_tlp_rtx_out = 0;
13112         } else {
13113                 /*
13114                  * Its a TLP.
13115                  */
13116                 bbr->rc_tlp_in_progress = 1;
13117         }
13118         bbr_timer_cancel(bbr, __LINE__, cts);
13119         if (rsm == NULL) {
13120                 if (sbused(sb) > 0) {
13121                         /*
13122                          * This is sub-optimal. We only send a stand alone
13123                          * FIN on its own segment.
13124                          */
13125                         if (flags & TH_FIN) {
13126                                 flags &= ~TH_FIN;
13127                                 if ((len == 0) && ((tp->t_flags & TF_ACKNOW) == 0)) {
13128                                         /* Lets not send this */
13129                                         slot = 0;
13130                                         goto just_return;
13131                                 }
13132                         }
13133                 }
13134         } else {
13135                 /*
13136                  * We do *not* send a FIN on a retransmit if it has data.
13137                  * The if clause here where len > 1 should never come true.
13138                  */
13139                 if ((len > 0) &&
13140                     (((rsm->r_flags & BBR_HAS_FIN) == 0) &&
13141                     (flags & TH_FIN))) {
13142                         flags &= ~TH_FIN;
13143                         len--;
13144                 }
13145         }
13146         SOCKBUF_LOCK_ASSERT(sb);
13147         if (len > 0) {
13148                 if ((tp->snd_una == tp->snd_max) &&
13149                     (bbr_calc_time(cts, bbr->r_ctl.rc_went_idle_time) >= bbr_rtt_probe_time)) {
13150                         /*
13151                          * This qualifies as a RTT_PROBE session since we
13152                          * drop the data outstanding to nothing and waited
13153                          * more than bbr_rtt_probe_time.
13154                          */
13155                         bbr_log_rtt_shrinks(bbr, cts, 0, 0, __LINE__, BBR_RTTS_WASIDLE, 0);
13156                         bbr_set_reduced_rtt(bbr, cts, __LINE__);
13157                 }
13158                 if (len >= maxseg)
13159                         tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;
13160                 else
13161                         tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;
13162         }
13163         /*
13164          * Before ESTABLISHED, force sending of initial options unless TCP
13165          * set not to do any options. NOTE: we assume that the IP/TCP header
13166          * plus TCP options always fit in a single mbuf, leaving room for a
13167          * maximum link header, i.e. max_linkhdr + sizeof (struct tcpiphdr)
13168          * + optlen <= MCLBYTES
13169          */
13170         optlen = 0;
13171 #ifdef INET6
13172         if (isipv6)
13173                 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
13174         else
13175 #endif
13176                 hdrlen = sizeof(struct tcpiphdr);
13177
13178         /*
13179          * Compute options for segment. We only have to care about SYN and
13180          * established connection segments.  Options for SYN-ACK segments
13181          * are handled in TCP syncache.
13182          */
13183         to.to_flags = 0;
13184         local_options = 0;
13185         if ((tp->t_flags & TF_NOOPT) == 0) {
13186                 /* Maximum segment size. */
13187                 if (flags & TH_SYN) {
13188                         to.to_mss = tcp_mssopt(&inp->inp_inc);
13189 #ifdef NETFLIX_TCPOUDP
13190                         if (tp->t_port)
13191                                 to.to_mss -= V_tcp_udp_tunneling_overhead;
13192 #endif
13193                         to.to_flags |= TOF_MSS;
13194                         /*
13195                          * On SYN or SYN|ACK transmits on TFO connections,
13196                          * only include the TFO option if it is not a
13197                          * retransmit, as the presence of the TFO option may
13198                          * have caused the original SYN or SYN|ACK to have
13199                          * been dropped by a middlebox.
13200                          */
13201                         if (IS_FASTOPEN(tp->t_flags) &&
13202                             (tp->t_rxtshift == 0)) {
13203                                 if (tp->t_state == TCPS_SYN_RECEIVED) {
13204                                         to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN;
13205                                         to.to_tfo_cookie =
13206                                             (u_int8_t *)&tp->t_tfo_cookie.server;
13207                                         to.to_flags |= TOF_FASTOPEN;
13208                                         wanted_cookie = 1;
13209                                 } else if (tp->t_state == TCPS_SYN_SENT) {
13210                                         to.to_tfo_len =
13211                                             tp->t_tfo_client_cookie_len;
13212                                         to.to_tfo_cookie =
13213                                             tp->t_tfo_cookie.client;
13214                                         to.to_flags |= TOF_FASTOPEN;
13215                                         wanted_cookie = 1;
13216                                 }
13217                         }
13218                 }
13219                 /* Window scaling. */
13220                 if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
13221                         to.to_wscale = tp->request_r_scale;
13222                         to.to_flags |= TOF_SCALE;
13223                 }
13224                 /* Timestamps. */
13225                 if ((tp->t_flags & TF_RCVD_TSTMP) ||
13226                     ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {
13227                         to.to_tsval =   tcp_tv_to_mssectick(&bbr->rc_tv) + tp->ts_offset;
13228                         to.to_tsecr = tp->ts_recent;
13229                         to.to_flags |= TOF_TS;
13230                         local_options += TCPOLEN_TIMESTAMP + 2;
13231                 }
13232                 /* Set receive buffer autosizing timestamp. */
13233                 if (tp->rfbuf_ts == 0 &&
13234                     (so->so_rcv.sb_flags & SB_AUTOSIZE))
13235                         tp->rfbuf_ts =  tcp_tv_to_mssectick(&bbr->rc_tv);
13236                 /* Selective ACK's. */
13237                 if (flags & TH_SYN)
13238                         to.to_flags |= TOF_SACKPERM;
13239                 else if (TCPS_HAVEESTABLISHED(tp->t_state) &&
13240                     tp->rcv_numsacks > 0) {
13241                         to.to_flags |= TOF_SACK;
13242                         to.to_nsacks = tp->rcv_numsacks;
13243                         to.to_sacks = (u_char *)tp->sackblks;
13244                 }
13245 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
13246                 /* TCP-MD5 (RFC2385). */
13247                 if (tp->t_flags & TF_SIGNATURE)
13248                         to.to_flags |= TOF_SIGNATURE;
13249 #endif                          /* TCP_SIGNATURE */
13250
13251                 /* Processing the options. */
13252                 hdrlen += (optlen = tcp_addoptions(&to, opt));
13253                 /*
13254                  * If we wanted a TFO option to be added, but it was unable
13255                  * to fit, ensure no data is sent.
13256                  */
13257                 if (IS_FASTOPEN(tp->t_flags) && wanted_cookie &&
13258                     !(to.to_flags & TOF_FASTOPEN))
13259                         len = 0;
13260         }
13261 #ifdef NETFLIX_TCPOUDP
13262         if (tp->t_port) {
13263                 if (V_tcp_udp_tunneling_port == 0) {
13264                         /* The port was removed?? */
13265                         SOCKBUF_UNLOCK(&so->so_snd);
13266                         return (EHOSTUNREACH);
13267                 }
13268                 hdrlen += sizeof(struct udphdr);
13269         }
13270 #endif
13271 #ifdef INET6
13272         if (isipv6)
13273                 ipoptlen = ip6_optlen(tp->t_inpcb);
13274         else
13275 #endif
13276         if (tp->t_inpcb->inp_options)
13277                 ipoptlen = tp->t_inpcb->inp_options->m_len -
13278                     offsetof(struct ipoption, ipopt_list);
13279         else
13280                 ipoptlen = 0;
13281         ipoptlen = 0;
13282 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
13283         ipoptlen += ipsec_optlen;
13284 #endif
13285         if (bbr->rc_last_options != local_options) {
13286                 /*
13287                  * Cache the options length this generally does not change
13288                  * on a connection. We use this to calculate TSO.
13289                  */
13290                 bbr->rc_last_options = local_options;
13291         }
13292         maxseg = tp->t_maxseg - (ipoptlen + optlen);
13293         p_maxseg = min(maxseg, pace_max_segs);
13294         /*
13295          * Adjust data length if insertion of options will bump the packet
13296          * length beyond the t_maxseg length. Clear the FIN bit because we
13297          * cut off the tail of the segment.
13298          */
13299 #ifdef KERN_TLS
13300         /* force TSO for so TLS offload can get mss */
13301         if (sb->sb_flags & SB_TLS_IFNET) {
13302                 force_tso = 1;
13303         }
13304 #endif
13305
13306         if (len > maxseg) {
13307                 if (len != 0 && (flags & TH_FIN)) {
13308                         flags &= ~TH_FIN;
13309                 }
13310                 if (tso) {
13311                         uint32_t moff;
13312                         int32_t max_len;
13313
13314                         /* extract TSO information */
13315                         if_hw_tsomax = tp->t_tsomax;
13316                         if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
13317                         if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
13318                         KASSERT(ipoptlen == 0,
13319                             ("%s: TSO can't do IP options", __func__));
13320
13321                         /*
13322                          * Check if we should limit by maximum payload
13323                          * length:
13324                          */
13325                         if (if_hw_tsomax != 0) {
13326                                 /* compute maximum TSO length */
13327                                 max_len = (if_hw_tsomax - hdrlen -
13328                                     max_linkhdr);
13329                                 if (max_len <= 0) {
13330                                         len = 0;
13331                                 } else if (len > max_len) {
13332                                         len = max_len;
13333                                 }
13334                         }
13335                         /*
13336                          * Prevent the last segment from being fractional
13337                          * unless the send sockbuf can be emptied:
13338                          */
13339                         if (((sb_offset + len) < sbavail(sb)) &&
13340                             (hw_tls == 0)) {
13341                                 moff = len % (uint32_t)maxseg;
13342                                 if (moff != 0) {
13343                                         len -= moff;
13344                                 }
13345                         }
13346                         /*
13347                          * In case there are too many small fragments don't
13348                          * use TSO:
13349                          */
13350                         if (len <= maxseg) {
13351                                 len = maxseg;
13352                                 tso = 0;
13353                         }
13354                 } else {
13355                         /* Not doing TSO */
13356                         if (optlen + ipoptlen >= tp->t_maxseg) {
13357                                 /*
13358                                  * Since we don't have enough space to put
13359                                  * the IP header chain and the TCP header in
13360                                  * one packet as required by RFC 7112, don't
13361                                  * send it. Also ensure that at least one
13362                                  * byte of the payload can be put into the
13363                                  * TCP segment.
13364                                  */
13365                                 SOCKBUF_UNLOCK(&so->so_snd);
13366                                 error = EMSGSIZE;
13367                                 sack_rxmit = 0;
13368                                 goto out;
13369                         }
13370                         len = maxseg;
13371                 }
13372         } else {
13373                 /* Not doing TSO */
13374                 if_hw_tsomaxsegcount = 0;
13375                 tso = 0;
13376         }
13377         KASSERT(len + hdrlen + ipoptlen <= IP_MAXPACKET,
13378             ("%s: len > IP_MAXPACKET", __func__));
13379 #ifdef DIAGNOSTIC
13380 #ifdef INET6
13381         if (max_linkhdr + hdrlen > MCLBYTES)
13382 #else
13383         if (max_linkhdr + hdrlen > MHLEN)
13384 #endif
13385                 panic("tcphdr too big");
13386 #endif
13387         /*
13388          * This KASSERT is here to catch edge cases at a well defined place.
13389          * Before, those had triggered (random) panic conditions further
13390          * down.
13391          */
13392 #ifdef BBR_INVARIANTS
13393         if (sack_rxmit) {
13394                 if (SEQ_LT(rsm->r_start, tp->snd_una)) {
13395                         panic("RSM:%p TP:%p bbr:%p start:%u is < snd_una:%u",
13396                             rsm, tp, bbr, rsm->r_start, tp->snd_una);
13397                 }
13398         }
13399 #endif
13400         KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
13401         if ((len == 0) &&
13402             (flags & TH_FIN) &&
13403             (sbused(sb))) {
13404                 /*
13405                  * We have outstanding data, don't send a fin by itself!.
13406                  */
13407                 slot = 0;
13408                 goto just_return;
13409         }
13410         /*
13411          * Grab a header mbuf, attaching a copy of data to be transmitted,
13412          * and initialize the header from the template for sends on this
13413          * connection.
13414          */
13415         if (len) {
13416                 uint32_t moff;
13417                 uint32_t orig_len;
13418
13419                 /*
13420                  * We place a limit on sending with hptsi.
13421                  */
13422                 if ((rsm == NULL) && len > pace_max_segs)
13423                         len = pace_max_segs;
13424                 if (len <= maxseg)
13425                         tso = 0;
13426 #ifdef INET6
13427                 if (MHLEN < hdrlen + max_linkhdr)
13428                         m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
13429                 else
13430 #endif
13431                         m = m_gethdr(M_NOWAIT, MT_DATA);
13432
13433                 if (m == NULL) {
13434                         BBR_STAT_INC(bbr_failed_mbuf_aloc);
13435                         bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0);
13436                         SOCKBUF_UNLOCK(sb);
13437                         error = ENOBUFS;
13438                         sack_rxmit = 0;
13439                         goto out;
13440                 }
13441                 m->m_data += max_linkhdr;
13442                 m->m_len = hdrlen;
13443                 /*
13444                  * Start the m_copy functions from the closest mbuf to the
13445                  * sb_offset in the socket buffer chain.
13446                  */
13447                 if ((sb_offset > sbavail(sb)) || ((len + sb_offset) > sbavail(sb))) {
13448 #ifdef BBR_INVARIANTS
13449                         if ((len + sb_offset) > (sbavail(sb) + ((flags & (TH_FIN | TH_SYN)) ? 1 : 0)))
13450                                 panic("tp:%p bbr:%p len:%u sb_offset:%u sbavail:%u rsm:%p %u:%u:%u",
13451                                     tp, bbr, len, sb_offset, sbavail(sb), rsm,
13452                                     doing_retran_from,
13453                                     picked_up_retran,
13454                                     doing_tlp);
13455
13456 #endif
13457                         /*
13458                          * In this messed up situation we have two choices,
13459                          * a) pretend the send worked, and just start timers
13460                          * and what not (not good since that may lead us
13461                          * back here a lot). <or> b) Send the lowest segment
13462                          * in the map. <or> c) Drop the connection. Lets do
13463                          * <b> which if it continues to happen will lead to
13464                          * <c> via timeouts.
13465                          */
13466                         BBR_STAT_INC(bbr_offset_recovery);
13467                         rsm = TAILQ_FIRST(&bbr->r_ctl.rc_map);
13468                         sb_offset = 0;
13469                         if (rsm == NULL) {
13470                                 sack_rxmit = 0;
13471                                 len = sbavail(sb);
13472                         } else {
13473                                 sack_rxmit = 1;
13474                                 if (rsm->r_start != tp->snd_una) {
13475                                         /*
13476                                          * Things are really messed up, <c>
13477                                          * is the only thing to do.
13478                                          */
13479                                         BBR_STAT_INC(bbr_offset_drop);
13480                                         tcp_set_inp_to_drop(inp, EFAULT);
13481                                         return (0);
13482                                 }
13483                                 len = rsm->r_end - rsm->r_start;
13484                         }
13485                         if (len > sbavail(sb))
13486                                 len = sbavail(sb);
13487                         if (len > maxseg)
13488                                 len = maxseg;
13489                 }
13490                 mb = sbsndptr_noadv(sb, sb_offset, &moff);
13491                 if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) {
13492                         m_copydata(mb, moff, (int)len,
13493                             mtod(m, caddr_t)+hdrlen);
13494                         if (rsm == NULL)
13495                                 sbsndptr_adv(sb, mb, len);
13496                         m->m_len += len;
13497                 } else {
13498                         struct sockbuf *msb;
13499
13500                         if (rsm)
13501                                 msb = NULL;
13502                         else
13503                                 msb = sb;
13504 #ifdef BBR_INVARIANTS
13505                         if ((len + moff) > (sbavail(sb) + ((flags & (TH_FIN | TH_SYN)) ? 1 : 0))) {
13506                                 if (rsm) {
13507                                         panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u rsm:%p snd_una:%u rsm_start:%u flg:%x %u:%u:%u sr:%d ",
13508                                             tp, bbr, len, moff,
13509                                             sbavail(sb), rsm,
13510                                             tp->snd_una, rsm->r_flags, rsm->r_start,
13511                                             doing_retran_from,
13512                                             picked_up_retran,
13513                                             doing_tlp, sack_rxmit);
13514                                 } else {
13515                                         panic("tp:%p bbr:%p len:%u moff:%u sbavail:%u sb_offset:%u snd_una:%u",
13516                                             tp, bbr, len, moff, sbavail(sb), sb_offset, tp->snd_una);
13517                                 }
13518                         }
13519 #endif
13520                         orig_len = len;
13521                         m->m_next = tcp_m_copym(
13522 #ifdef NETFLIX_COPY_ARGS
13523                                 tp,
13524 #endif
13525                                 mb, moff, &len,
13526                                 if_hw_tsomaxsegcount,
13527                                 if_hw_tsomaxsegsize, msb,
13528                                 ((rsm == NULL) ? hw_tls : 0)
13529 #ifdef NETFLIX_COPY_ARGS
13530                                 , &filled_all
13531 #endif
13532                                 );
13533                         if (len <= maxseg && !force_tso) {
13534                                 /*
13535                                  * Must have ran out of mbufs for the copy
13536                                  * shorten it to no longer need tso. Lets
13537                                  * not put on sendalot since we are low on
13538                                  * mbufs.
13539                                  */
13540                                 tso = 0;
13541                         }
13542                         if (m->m_next == NULL) {
13543                                 SOCKBUF_UNLOCK(sb);
13544                                 (void)m_free(m);
13545                                 error = ENOBUFS;
13546                                 sack_rxmit = 0;
13547                                 goto out;
13548                         }
13549                 }
13550 #ifdef BBR_INVARIANTS
13551                 if (tso && len < maxseg) {
13552                         panic("tp:%p tso on, but len:%d < maxseg:%d",
13553                             tp, len, maxseg);
13554                 }
13555                 if (tso && if_hw_tsomaxsegcount) {
13556                         int32_t seg_cnt = 0;
13557                         struct mbuf *foo;
13558
13559                         foo = m;
13560                         while (foo) {
13561                                 seg_cnt++;
13562                                 foo = foo->m_next;
13563                         }
13564                         if (seg_cnt > if_hw_tsomaxsegcount) {
13565                                 panic("seg_cnt:%d > max:%d", seg_cnt, if_hw_tsomaxsegcount);
13566                         }
13567                 }
13568 #endif
13569                 /*
13570                  * If we're sending everything we've got, set PUSH. (This
13571                  * will keep happy those implementations which only give
13572                  * data to the user when a buffer fills or a PUSH comes in.)
13573                  */
13574                 if (sb_offset + len == sbused(sb) &&
13575                     sbused(sb) &&
13576                     !(flags & TH_SYN)) {
13577                         flags |= TH_PUSH;
13578                 }
13579                 SOCKBUF_UNLOCK(sb);
13580         } else {
13581                 SOCKBUF_UNLOCK(sb);
13582                 if (tp->t_flags & TF_ACKNOW)
13583                         KMOD_TCPSTAT_INC(tcps_sndacks);
13584                 else if (flags & (TH_SYN | TH_FIN | TH_RST))
13585                         KMOD_TCPSTAT_INC(tcps_sndctrl);
13586                 else if (SEQ_GT(tp->snd_up, tp->snd_una))
13587                         KMOD_TCPSTAT_INC(tcps_sndurg);
13588                 else
13589                         KMOD_TCPSTAT_INC(tcps_sndwinup);
13590
13591                 m = m_gethdr(M_NOWAIT, MT_DATA);
13592                 if (m == NULL) {
13593                         BBR_STAT_INC(bbr_failed_mbuf_aloc);
13594                         bbr_log_enobuf_jmp(bbr, len, cts, __LINE__, len, 0, 0);
13595                         error = ENOBUFS;
13596                         /* Fudge the send time since we could not send */
13597                         sack_rxmit = 0;
13598                         goto out;
13599                 }
13600 #ifdef INET6
13601                 if (isipv6 && (MHLEN < hdrlen + max_linkhdr) &&
13602                     MHLEN >= hdrlen) {
13603                         M_ALIGN(m, hdrlen);
13604                 } else
13605 #endif
13606                         m->m_data += max_linkhdr;
13607                 m->m_len = hdrlen;
13608         }
13609         SOCKBUF_UNLOCK_ASSERT(sb);
13610         m->m_pkthdr.rcvif = (struct ifnet *)0;
13611 #ifdef MAC
13612         mac_inpcb_create_mbuf(inp, m);
13613 #endif
13614 #ifdef INET6
13615         if (isipv6) {
13616                 ip6 = mtod(m, struct ip6_hdr *);
13617 #ifdef NETFLIX_TCPOUDP
13618                 if (tp->t_port) {
13619                         udp = (struct udphdr *)((caddr_t)ip6 + ipoptlen + sizeof(struct ip6_hdr));
13620                         udp->uh_sport = htons(V_tcp_udp_tunneling_port);
13621                         udp->uh_dport = tp->t_port;
13622                         ulen = hdrlen + len - sizeof(struct ip6_hdr);
13623                         udp->uh_ulen = htons(ulen);
13624                         th = (struct tcphdr *)(udp + 1);
13625                 } else {
13626 #endif
13627                         th = (struct tcphdr *)(ip6 + 1);
13628
13629 #ifdef NETFLIX_TCPOUDP
13630                 }
13631 #endif
13632                 tcpip_fillheaders(inp,
13633 #ifdef NETFLIX_TCPOUDP
13634                                   tp->t_port,
13635 #endif
13636                                   ip6, th);
13637         } else
13638 #endif                          /* INET6 */
13639         {
13640                 ip = mtod(m, struct ip *);
13641 #ifdef TCPDEBUG
13642                 ipov = (struct ipovly *)ip;
13643 #endif
13644 #ifdef NETFLIX_TCPOUDP
13645                 if (tp->t_port) {
13646                         udp = (struct udphdr *)((caddr_t)ip + ipoptlen + sizeof(struct ip));
13647                         udp->uh_sport = htons(V_tcp_udp_tunneling_port);
13648                         udp->uh_dport = tp->t_port;
13649                         ulen = hdrlen + len - sizeof(struct ip);
13650                         udp->uh_ulen = htons(ulen);
13651                         th = (struct tcphdr *)(udp + 1);
13652                 } else
13653 #endif
13654                         th = (struct tcphdr *)(ip + 1);
13655                 tcpip_fillheaders(inp,
13656 #ifdef NETFLIX_TCPOUDP
13657                                   tp->t_port,
13658 #endif
13659                                   ip, th);
13660         }
13661         /*
13662          * If we are doing retransmissions, then snd_nxt will not reflect
13663          * the first unsent octet.  For ACK only packets, we do not want the
13664          * sequence number of the retransmitted packet, we want the sequence
13665          * number of the next unsent octet.  So, if there is no data (and no
13666          * SYN or FIN), use snd_max instead of snd_nxt when filling in
13667          * ti_seq.  But if we are in persist state, snd_max might reflect
13668          * one byte beyond the right edge of the window, so use snd_nxt in
13669          * that case, since we know we aren't doing a retransmission.
13670          * (retransmit and persist are mutually exclusive...)
13671          */
13672         if (sack_rxmit == 0) {
13673                 if (len && ((flags & (TH_FIN | TH_SYN | TH_RST)) == 0)) {
13674                         /* New data (including new persists) */
13675                         th->th_seq = htonl(tp->snd_max);
13676                         bbr_seq = tp->snd_max;
13677                 } else if (flags & TH_SYN) {
13678                         /* Syn's always send from iss */
13679                         th->th_seq = htonl(tp->iss);
13680                         bbr_seq = tp->iss;
13681                 } else if (flags & TH_FIN) {
13682                         if (flags & TH_FIN && tp->t_flags & TF_SENTFIN) {
13683                                 /*
13684                                  * If we sent the fin already its 1 minus
13685                                  * snd_max
13686                                  */
13687                                 th->th_seq = (htonl(tp->snd_max - 1));
13688                                 bbr_seq = (tp->snd_max - 1);
13689                         } else {
13690                                 /* First time FIN use snd_max */
13691                                 th->th_seq = htonl(tp->snd_max);
13692                                 bbr_seq = tp->snd_max;
13693                         }
13694                 } else if (flags & TH_RST) {
13695                         /*
13696                          * For a Reset send the last cum ack in sequence
13697                          * (this like any other choice may still generate a
13698                          * challenge ack, if a ack-update packet is in
13699                          * flight).
13700                          */
13701                         th->th_seq = htonl(tp->snd_una);
13702                         bbr_seq = tp->snd_una;
13703                 } else {
13704                         /*
13705                          * len == 0 and not persist we use snd_max, sending
13706                          * an ack unless we have sent the fin then its 1
13707                          * minus.
13708                          */
13709                         /*
13710                          * XXXRRS Question if we are in persists and we have
13711                          * nothing outstanding to send and we have not sent
13712                          * a FIN, we will send an ACK. In such a case it
13713                          * might be better to send (tp->snd_una - 1) which
13714                          * would force the peer to ack.
13715                          */
13716                         if (tp->t_flags & TF_SENTFIN) {
13717                                 th->th_seq = htonl(tp->snd_max - 1);
13718                                 bbr_seq = (tp->snd_max - 1);
13719                         } else {
13720                                 th->th_seq = htonl(tp->snd_max);
13721                                 bbr_seq = tp->snd_max;
13722                         }
13723                 }
13724         } else {
13725                 /* All retransmits use the rsm to guide the send */
13726                 th->th_seq = htonl(rsm->r_start);
13727                 bbr_seq = rsm->r_start;
13728         }
13729         th->th_ack = htonl(tp->rcv_nxt);
13730         if (optlen) {
13731                 bcopy(opt, th + 1, optlen);
13732                 th->th_off = (sizeof(struct tcphdr) + optlen) >> 2;
13733         }
13734         th->th_flags = flags;
13735         /*
13736          * Calculate receive window.  Don't shrink window, but avoid silly
13737          * window syndrome.
13738          */
13739         if ((flags & TH_RST) || ((recwin < (so->so_rcv.sb_hiwat / 4) &&
13740                                   recwin < maxseg)))
13741                 recwin = 0;
13742         if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&
13743             recwin < (tp->rcv_adv - tp->rcv_nxt))
13744                 recwin = (tp->rcv_adv - tp->rcv_nxt);
13745         if (recwin > TCP_MAXWIN << tp->rcv_scale)
13746                 recwin = TCP_MAXWIN << tp->rcv_scale;
13747
13748         /*
13749          * According to RFC1323 the window field in a SYN (i.e., a <SYN> or
13750          * <SYN,ACK>) segment itself is never scaled.  The <SYN,ACK> case is
13751          * handled in syncache.
13752          */
13753         if (flags & TH_SYN)
13754                 th->th_win = htons((u_short)
13755                     (min(sbspace(&so->so_rcv), TCP_MAXWIN)));
13756         else
13757                 th->th_win = htons((u_short)(recwin >> tp->rcv_scale));
13758         /*
13759          * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0
13760          * window.  This may cause the remote transmitter to stall.  This
13761          * flag tells soreceive() to disable delayed acknowledgements when
13762          * draining the buffer.  This can occur if the receiver is
13763          * attempting to read more data than can be buffered prior to
13764          * transmitting on the connection.
13765          */
13766         if (th->th_win == 0) {
13767                 tp->t_sndzerowin++;
13768                 tp->t_flags |= TF_RXWIN0SENT;
13769         } else
13770                 tp->t_flags &= ~TF_RXWIN0SENT;
13771         if (SEQ_GT(tp->snd_up, tp->snd_max)) {
13772                 th->th_urp = htons((u_short)(tp->snd_up - tp->snd_max));
13773                 th->th_flags |= TH_URG;
13774         } else
13775                 /*
13776                  * If no urgent pointer to send, then we pull the urgent
13777                  * pointer to the left edge of the send window so that it
13778                  * doesn't drift into the send window on sequence number
13779                  * wraparound.
13780                  */
13781                 tp->snd_up = tp->snd_una;       /* drag it along */
13782
13783 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
13784         if (to.to_flags & TOF_SIGNATURE) {
13785                 /*
13786                  * Calculate MD5 signature and put it into the place
13787                  * determined before. NOTE: since TCP options buffer doesn't
13788                  * point into mbuf's data, calculate offset and use it.
13789                  */
13790                 if (!TCPMD5_ENABLED() || TCPMD5_OUTPUT(m, th,
13791                     (u_char *)(th + 1) + (to.to_signature - opt)) != 0) {
13792                         /*
13793                          * Do not send segment if the calculation of MD5
13794                          * digest has failed.
13795                          */
13796                         goto out;
13797                 }
13798         }
13799 #endif
13800
13801         /*
13802          * Put TCP length in extended header, and then checksum extended
13803          * header and data.
13804          */
13805         m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
13806 #ifdef INET6
13807         if (isipv6) {
13808                 /*
13809                  * ip6_plen is not need to be filled now, and will be filled
13810                  * in ip6_output.
13811                  */
13812 #ifdef NETFLIX_TCPOUDP
13813                 if (tp->t_port) {
13814                         m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
13815                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13816                         udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
13817                         th->th_sum = htons(0);
13818                         UDPSTAT_INC(udps_opackets);
13819                 } else {
13820 #endif
13821                         csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
13822                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13823                         th->th_sum = in6_cksum_pseudo(ip6, sizeof(struct tcphdr) +
13824                             optlen + len, IPPROTO_TCP, 0);
13825 #ifdef NETFLIX_TCPOUDP
13826                 }
13827 #endif
13828         }
13829 #endif
13830 #if defined(INET6) && defined(INET)
13831         else
13832 #endif
13833 #ifdef INET
13834         {
13835 #ifdef NETFLIX_TCPOUDP
13836                 if (tp->t_port) {
13837                         m->m_pkthdr.csum_flags = CSUM_UDP;
13838                         m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13839                         udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
13840                             ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
13841                         th->th_sum = htons(0);
13842                         UDPSTAT_INC(udps_opackets);
13843                 } else {
13844 #endif
13845                         csum_flags = m->m_pkthdr.csum_flags = CSUM_TCP;
13846                         m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13847                         th->th_sum = in_pseudo(ip->ip_src.s_addr,
13848                             ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
13849                             IPPROTO_TCP + len + optlen));
13850 #ifdef NETFLIX_TCPOUDP
13851                 }
13852 #endif
13853                 /* IP version must be set here for ipv4/ipv6 checking later */
13854                 KASSERT(ip->ip_v == IPVERSION,
13855                     ("%s: IP version incorrect: %d", __func__, ip->ip_v));
13856         }
13857 #endif
13858
13859         /*
13860          * Enable TSO and specify the size of the segments. The TCP pseudo
13861          * header checksum is always provided. XXX: Fixme: This is currently
13862          * not the case for IPv6.
13863          */
13864         if (tso || force_tso) {
13865                 KASSERT(force_tso || len > maxseg,
13866                     ("%s: len:%d <= tso_segsz:%d", __func__, len, maxseg));
13867                 m->m_pkthdr.csum_flags |= CSUM_TSO;
13868                 csum_flags |= CSUM_TSO;
13869                 m->m_pkthdr.tso_segsz = maxseg;
13870         }
13871         KASSERT(len + hdrlen == m_length(m, NULL),
13872             ("%s: mbuf chain different than expected: %d + %u != %u",
13873             __func__, len, hdrlen, m_length(m, NULL)));
13874
13875 #ifdef TCP_HHOOK
13876         /* Run HHOOK_TC_ESTABLISHED_OUT helper hooks. */
13877         hhook_run_tcp_est_out(tp, th, &to, len, tso);
13878 #endif
13879 #ifdef TCPDEBUG
13880         /*
13881          * Trace.
13882          */
13883         if (so->so_options & SO_DEBUG) {
13884                 u_short save = 0;
13885
13886 #ifdef INET6
13887                 if (!isipv6)
13888 #endif
13889                 {
13890                         save = ipov->ih_len;
13891                         ipov->ih_len = htons(m->m_pkthdr.len    /* - hdrlen +
13892                               * (th->th_off << 2) */ );
13893                 }
13894                 tcp_trace(TA_OUTPUT, tp->t_state, tp, mtod(m, void *), th, 0);
13895 #ifdef INET6
13896                 if (!isipv6)
13897 #endif
13898                         ipov->ih_len = save;
13899         }
13900 #endif                          /* TCPDEBUG */
13901
13902         /* Log to the black box */
13903         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
13904                 union tcp_log_stackspecific log;
13905
13906                 bbr_fill_in_logging_data(bbr, &log.u_bbr, cts);
13907                 /* Record info on type of transmission */
13908                 log.u_bbr.flex1 = bbr->r_ctl.rc_hptsi_agg_delay;
13909                 log.u_bbr.flex2 = (bbr->r_recovery_bw << 3);
13910                 log.u_bbr.flex3 = maxseg;
13911                 log.u_bbr.flex4 = delay_calc;
13912                 /* Encode filled_all into the upper flex5 bit */
13913                 log.u_bbr.flex5 = bbr->rc_past_init_win;
13914                 log.u_bbr.flex5 <<= 1;
13915                 log.u_bbr.flex5 |= bbr->rc_no_pacing;
13916                 log.u_bbr.flex5 <<= 29;
13917                 if (filled_all)
13918                         log.u_bbr.flex5 |= 0x80000000;
13919                 log.u_bbr.flex5 |= tp->t_maxseg;
13920                 log.u_bbr.flex6 = bbr->r_ctl.rc_pace_max_segs;
13921                 log.u_bbr.flex7 = (bbr->rc_bbr_state << 8) | bbr_state_val(bbr);
13922                 /* lets poke in the low and the high here for debugging */
13923                 log.u_bbr.pkts_out = bbr->rc_tp->t_maxseg;
13924                 if (rsm || sack_rxmit) {
13925                         if (doing_tlp)
13926                                 log.u_bbr.flex8 = 2;
13927                         else
13928                                 log.u_bbr.flex8 = 1;
13929                 } else {
13930                         log.u_bbr.flex8 = 0;
13931                 }
13932                 lgb = tcp_log_event_(tp, th, &so->so_rcv, &so->so_snd, TCP_LOG_OUT, ERRNO_UNK,
13933                     len, &log, false, NULL, NULL, 0, tv);
13934         } else {
13935                 lgb = NULL;
13936         }
13937         /*
13938          * Fill in IP length and desired time to live and send to IP level.
13939          * There should be a better way to handle ttl and tos; we could keep
13940          * them in the template, but need a way to checksum without them.
13941          */
13942         /*
13943          * m->m_pkthdr.len should have been set before cksum calcuration,
13944          * because in6_cksum() need it.
13945          */
13946 #ifdef INET6
13947         if (isipv6) {
13948                 /*
13949                  * we separately set hoplimit for every segment, since the
13950                  * user might want to change the value via setsockopt. Also,
13951                  * desired default hop limit might be changed via Neighbor
13952                  * Discovery.
13953                  */
13954                 ip6->ip6_hlim = in6_selecthlim(inp, NULL);
13955
13956                 /*
13957                  * Set the packet size here for the benefit of DTrace
13958                  * probes. ip6_output() will set it properly; it's supposed
13959                  * to include the option header lengths as well.
13960                  */
13961                 ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
13962
13963                 if (V_path_mtu_discovery && maxseg > V_tcp_minmss)
13964                         tp->t_flags2 |= TF2_PLPMTU_PMTUD;
13965                 else
13966                         tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
13967
13968                 if (tp->t_state == TCPS_SYN_SENT)
13969                         TCP_PROBE5(connect__request, NULL, tp, ip6, tp, th);
13970
13971                 TCP_PROBE5(send, NULL, tp, ip6, tp, th);
13972                 /* TODO: IPv6 IP6TOS_ECT bit on */
13973                 error = ip6_output(m, inp->in6p_outputopts,
13974                     &inp->inp_route6,
13975                     ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0),
13976                     NULL, NULL, inp);
13977
13978                 if (error == EMSGSIZE && inp->inp_route6.ro_rt != NULL)
13979                         mtu = inp->inp_route6.ro_rt->rt_mtu;
13980         }
13981 #endif                          /* INET6 */
13982 #if defined(INET) && defined(INET6)
13983         else
13984 #endif
13985 #ifdef INET
13986         {
13987                 ip->ip_len = htons(m->m_pkthdr.len);
13988 #ifdef INET6
13989                 if (isipv6)
13990                         ip->ip_ttl = in6_selecthlim(inp, NULL);
13991 #endif                          /* INET6 */
13992                 /*
13993                  * If we do path MTU discovery, then we set DF on every
13994                  * packet. This might not be the best thing to do according
13995                  * to RFC3390 Section 2. However the tcp hostcache migitates
13996                  * the problem so it affects only the first tcp connection
13997                  * with a host.
13998                  *
13999                  * NB: Don't set DF on small MTU/MSS to have a safe
14000                  * fallback.
14001                  */
14002                 if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
14003                         tp->t_flags2 |= TF2_PLPMTU_PMTUD;
14004                         if (tp->t_port == 0 || len < V_tcp_minmss) {
14005                                 ip->ip_off |= htons(IP_DF);
14006                         }
14007                 } else {
14008                         tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
14009                 }
14010
14011                 if (tp->t_state == TCPS_SYN_SENT)
14012                         TCP_PROBE5(connect__request, NULL, tp, ip, tp, th);
14013
14014                 TCP_PROBE5(send, NULL, tp, ip, tp, th);
14015
14016                 error = ip_output(m, inp->inp_options, &inp->inp_route,
14017                     ((rsm || sack_rxmit) ? IP_NO_SND_TAG_RL : 0), 0,
14018                     inp);
14019                 if (error == EMSGSIZE && inp->inp_route.ro_rt != NULL)
14020                         mtu = inp->inp_route.ro_rt->rt_mtu;
14021         }
14022 #endif                          /* INET */
14023 out:
14024
14025         if (lgb) {
14026                 lgb->tlb_errno = error;
14027                 lgb = NULL;
14028         }
14029         /*
14030          * In transmit state, time the transmission and arrange for the
14031          * retransmit.  In persist state, just set snd_max.
14032          */
14033         if (error == 0) {
14034                 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
14035                     (tp->t_flags & TF_SACK_PERMIT) &&
14036                     tp->rcv_numsacks > 0)
14037                         tcp_clean_dsack_blocks(tp);
14038                 /* We sent an ack clear the bbr_segs_rcvd count */
14039                 bbr->output_error_seen = 0;
14040                 bbr->oerror_cnt = 0;
14041                 bbr->bbr_segs_rcvd = 0;
14042                 if (len == 0)
14043                         counter_u64_add(bbr_out_size[TCP_MSS_ACCT_SNDACK], 1);
14044                 else if (hw_tls) {
14045                         if (filled_all ||
14046                             (len >= bbr->r_ctl.rc_pace_max_segs))
14047                                 BBR_STAT_INC(bbr_meets_tso_thresh);
14048                         else {
14049                                 if (doing_tlp) {
14050                                         BBR_STAT_INC(bbr_miss_tlp);
14051                                         bbr_log_type_hrdwtso(tp, bbr, len, 1, what_we_can);
14052
14053
14054                                 } else if (rsm) {
14055                                         BBR_STAT_INC(bbr_miss_retran);
14056                                         bbr_log_type_hrdwtso(tp, bbr, len, 2, what_we_can);
14057                                 } else if ((ctf_outstanding(tp) + bbr->r_ctl.rc_pace_max_segs) > sbavail(sb)) {
14058                                         BBR_STAT_INC(bbr_miss_tso_app);
14059                                         bbr_log_type_hrdwtso(tp, bbr, len, 3, what_we_can);
14060                                 } else if ((ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
14061                                                                  bbr->r_ctl.rc_lost_bytes)) + bbr->r_ctl.rc_pace_max_segs) > tp->snd_cwnd) {
14062                                         BBR_STAT_INC(bbr_miss_tso_cwnd);
14063                                         bbr_log_type_hrdwtso(tp, bbr, len, 4, what_we_can);
14064                                 } else if ((ctf_outstanding(tp) + bbr->r_ctl.rc_pace_max_segs) > tp->snd_wnd) {
14065                                         BBR_STAT_INC(bbr_miss_tso_rwnd);
14066                                         bbr_log_type_hrdwtso(tp, bbr, len, 5, what_we_can);
14067                                 } else {
14068                                         BBR_STAT_INC(bbr_miss_unknown);
14069                                         bbr_log_type_hrdwtso(tp, bbr, len, 6, what_we_can);
14070                                 }
14071                         }
14072                 }
14073                 /* Do accounting for new sends */
14074                 if ((len > 0) && (rsm == NULL)) {
14075                         int idx;
14076                         if (tp->snd_una == tp->snd_max) {
14077                                 /*
14078                                  * Special case to match google, when
14079                                  * nothing is in flight the delivered
14080                                  * time does get updated to the current
14081                                  * time (see tcp_rate_bsd.c).
14082                                  */
14083                                 bbr->r_ctl.rc_del_time = cts;
14084                         }
14085                         if (len >= maxseg) {
14086                                 idx = (len / maxseg) + 3;
14087                                 if (idx >= TCP_MSS_ACCT_ATIMER)
14088                                         counter_u64_add(bbr_out_size[(TCP_MSS_ACCT_ATIMER - 1)], 1);
14089                                 else
14090                                         counter_u64_add(bbr_out_size[idx], 1);
14091                         } else {
14092                                 /* smaller than a MSS */
14093                                 idx = len / (bbr_hptsi_bytes_min - bbr->rc_last_options);
14094                                 if (idx >= TCP_MSS_SMALL_MAX_SIZE_DIV)
14095                                         idx = (TCP_MSS_SMALL_MAX_SIZE_DIV - 1);
14096                                 counter_u64_add(bbr_out_size[(idx + TCP_MSS_SMALL_SIZE_OFF)], 1);
14097                         }
14098                 }
14099         }
14100         abandon = 0;
14101         /*
14102          * We must do the send accounting before we log the output,
14103          * otherwise the state of the rsm could change and we account to the
14104          * wrong bucket.
14105          */
14106         if (len > 0) {
14107                 bbr_do_send_accounting(tp, bbr, rsm, len, error);
14108                 if (error == 0) {
14109                         if (tp->snd_una == tp->snd_max)
14110                                 bbr->r_ctl.rc_tlp_rxt_last_time = cts;
14111                 }
14112         }
14113         bbr_log_output(bbr, tp, &to, len, bbr_seq, (uint8_t) flags, error,
14114             cts, mb, &abandon, rsm, 0, sb);
14115         if (abandon) {
14116                 /*
14117                  * If bbr_log_output destroys the TCB or sees a TH_RST being
14118                  * sent we should hit this condition.
14119                  */
14120                 return (0);
14121         }
14122         if (((tp->t_flags & TF_FORCEDATA) == 0) ||
14123             (bbr->rc_in_persist == 0)) {
14124                 /*
14125                  * Advance snd_nxt over sequence space of this segment.
14126                  */
14127                 if (error)
14128                         /* We don't log or do anything with errors */
14129                         goto skip_upd;
14130
14131                 if (tp->snd_una == tp->snd_max &&
14132                     (len || (flags & (TH_SYN | TH_FIN)))) {
14133                         /*
14134                          * Update the time we just added data since none was
14135                          * outstanding.
14136                          */
14137                         bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__);
14138                         bbr->rc_tp->t_acktime  = ticks;
14139                 }
14140                 if (flags & (TH_SYN | TH_FIN) && (rsm == NULL)) {
14141                         if (flags & TH_SYN) {
14142                                 tp->snd_max++;
14143                         }
14144                         if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {
14145                                 tp->snd_max++;
14146                                 tp->t_flags |= TF_SENTFIN;
14147                         }
14148                 }
14149                 if (sack_rxmit == 0)
14150                         tp->snd_max += len;
14151 skip_upd:
14152                 if ((error == 0) && len)
14153                         tot_len += len;
14154         } else {
14155                 /* Persists case */
14156                 int32_t xlen = len;
14157
14158                 if (error)
14159                         goto nomore;
14160
14161                 if (flags & TH_SYN)
14162                         ++xlen;
14163                 if ((flags & TH_FIN) && ((tp->t_flags & TF_SENTFIN) == 0)) {
14164                         ++xlen;
14165                         tp->t_flags |= TF_SENTFIN;
14166                 }
14167                 if (xlen && (tp->snd_una == tp->snd_max)) {
14168                         /*
14169                          * Update the time we just added data since none was
14170                          * outstanding.
14171                          */
14172                         bbr_log_progress_event(bbr, tp, ticks, PROGRESS_START, __LINE__);
14173                         bbr->rc_tp->t_acktime = ticks;
14174                 }
14175                 if (sack_rxmit == 0)
14176                         tp->snd_max += xlen;
14177                 tot_len += (len + optlen + ipoptlen);
14178         }
14179 nomore:
14180         if (error) {
14181                 /*
14182                  * Failures do not advance the seq counter above. For the
14183                  * case of ENOBUFS we will fall out and become ack-clocked.
14184                  * capping the cwnd at the current flight.
14185                  * Everything else will just have to retransmit with the timer
14186                  * (no pacer).
14187                  */
14188                 SOCKBUF_UNLOCK_ASSERT(sb);
14189                 BBR_STAT_INC(bbr_saw_oerr);
14190                 /* Clear all delay/early tracks */
14191                 bbr->r_ctl.rc_hptsi_agg_delay = 0;
14192                 bbr->r_ctl.rc_agg_early = 0;
14193                 bbr->r_agg_early_set = 0;
14194                 bbr->output_error_seen = 1;
14195                 if (bbr->oerror_cnt < 0xf)
14196                         bbr->oerror_cnt++;
14197                 if (bbr_max_net_error_cnt && (bbr->oerror_cnt >= bbr_max_net_error_cnt)) {
14198                         /* drop the session */
14199                         tcp_set_inp_to_drop(inp, ENETDOWN);
14200                 }
14201                 switch (error) {
14202                 case ENOBUFS:
14203                         /*
14204                          * Make this guy have to get ack's to send
14205                          * more but lets make sure we don't
14206                          * slam him below a T-O (1MSS).
14207                          */
14208                         if (bbr->rc_bbr_state != BBR_STATE_PROBE_RTT) {
14209                                 tp->snd_cwnd = ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
14210                                                                     bbr->r_ctl.rc_lost_bytes)) - maxseg;
14211                                 if (tp->snd_cwnd < maxseg)
14212                                         tp->snd_cwnd = maxseg;
14213                         }
14214                         slot = (bbr_error_base_paceout + 1) << bbr->oerror_cnt;
14215                         BBR_STAT_INC(bbr_saw_enobuf);
14216                         if (bbr->bbr_hdrw_pacing)
14217                                 counter_u64_add(bbr_hdwr_pacing_enobuf, 1);
14218                         else
14219                                 counter_u64_add(bbr_nohdwr_pacing_enobuf, 1);
14220                         /*
14221                          * Here even in the enobuf's case we want to do our
14222                          * state update. The reason being we may have been
14223                          * called by the input function. If so we have had
14224                          * things change.
14225                          */
14226                         error = 0;
14227                         goto enobufs;
14228                 case EMSGSIZE:
14229                         /*
14230                          * For some reason the interface we used initially
14231                          * to send segments changed to another or lowered
14232                          * its MTU. If TSO was active we either got an
14233                          * interface without TSO capabilits or TSO was
14234                          * turned off. If we obtained mtu from ip_output()
14235                          * then update it and try again.
14236                          */
14237                         /* Turn on tracing (or try to) */
14238                         {
14239                                 int old_maxseg;
14240
14241                                 old_maxseg = tp->t_maxseg;
14242                                 BBR_STAT_INC(bbr_saw_emsgsiz);
14243                                 bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, csum_flags, tso, cts);
14244                                 if (mtu != 0)
14245                                         tcp_mss_update(tp, -1, mtu, NULL, NULL);
14246                                 if (old_maxseg <= tp->t_maxseg) {
14247                                         /* Huh it did not shrink? */
14248                                         tp->t_maxseg = old_maxseg - 40;
14249                                         bbr_log_msgsize_fail(bbr, tp, len, maxseg, mtu, 0, tso, cts);
14250                                 }
14251                                 tp->t_flags &= ~TF_FORCEDATA;
14252                                 /*
14253                                  * Nuke all other things that can interfere
14254                                  * with slot
14255                                  */
14256                                 if ((tot_len + len) && (len >= tp->t_maxseg)) {
14257                                         slot = bbr_get_pacing_delay(bbr,
14258                                             bbr->r_ctl.rc_bbr_hptsi_gain,
14259                                             (tot_len + len), cts, 0);
14260                                         if (slot < bbr_error_base_paceout)
14261                                                 slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
14262                                 } else
14263                                         slot = (bbr_error_base_paceout + 2) << bbr->oerror_cnt;
14264                                 bbr->rc_output_starts_timer = 1;
14265                                 bbr_start_hpts_timer(bbr, tp, cts, 10, slot,
14266                                     tot_len);
14267                                 return (error);
14268                         }
14269                 case EPERM:
14270                         tp->t_softerror = error;
14271                         /* Fall through */
14272                 case EHOSTDOWN:
14273                 case EHOSTUNREACH:
14274                 case ENETDOWN:
14275                 case ENETUNREACH:
14276                         if (TCPS_HAVERCVDSYN(tp->t_state)) {
14277                                 tp->t_softerror = error;
14278                         }
14279                         /* FALLTHROUGH */
14280                 default:
14281                         tp->t_flags &= ~TF_FORCEDATA;
14282                         slot = (bbr_error_base_paceout + 3) << bbr->oerror_cnt;
14283                         bbr->rc_output_starts_timer = 1;
14284                         bbr_start_hpts_timer(bbr, tp, cts, 11, slot, 0);
14285                         return (error);
14286                 }
14287 #ifdef STATS
14288         } else if (((tp->t_flags & TF_GPUTINPROG) == 0) &&
14289                     len &&
14290                     (rsm == NULL) &&
14291             (bbr->rc_in_persist == 0)) {
14292                 tp->gput_seq = bbr_seq;
14293                 tp->gput_ack = bbr_seq +
14294                     min(sbavail(&so->so_snd) - sb_offset, sendwin);
14295                 tp->gput_ts = cts;
14296                 tp->t_flags |= TF_GPUTINPROG;
14297 #endif
14298         }
14299         KMOD_TCPSTAT_INC(tcps_sndtotal);
14300         if ((bbr->bbr_hdw_pace_ena) &&
14301             (bbr->bbr_attempt_hdwr_pace == 0) &&
14302             (bbr->rc_past_init_win) &&
14303             (bbr->rc_bbr_state != BBR_STATE_STARTUP) &&
14304             (get_filter_value(&bbr->r_ctl.rc_delrate)) &&
14305             (inp->inp_route.ro_rt &&
14306              inp->inp_route.ro_rt->rt_ifp)) {
14307                 /*
14308                  * We are past the initial window and
14309                  * have at least one measurement so we
14310                  * could use hardware pacing if its available.
14311                  * We have an interface and we have not attempted
14312                  * to setup hardware pacing, lets try to now.
14313                  */
14314                 uint64_t rate_wanted;
14315                 int err = 0;
14316
14317                 rate_wanted = bbr_get_hardware_rate(bbr);
14318                 bbr->bbr_attempt_hdwr_pace = 1;
14319                 bbr->r_ctl.crte = tcp_set_pacing_rate(bbr->rc_tp,
14320                                                       inp->inp_route.ro_rt->rt_ifp,
14321                                                       rate_wanted,
14322                                                       (RS_PACING_GEQ|RS_PACING_SUB_OK),
14323                                                       &err);
14324                 if (bbr->r_ctl.crte) {
14325                         bbr_type_log_hdwr_pacing(bbr,
14326                                                  bbr->r_ctl.crte->ptbl->rs_ifp,
14327                                                  rate_wanted,
14328                                                  bbr->r_ctl.crte->rate,
14329                                                  __LINE__, cts, err);
14330                         BBR_STAT_INC(bbr_hdwr_rl_add_ok);
14331                         counter_u64_add(bbr_flows_nohdwr_pacing, -1);
14332                         counter_u64_add(bbr_flows_whdwr_pacing, 1);
14333                         bbr->bbr_hdrw_pacing = 1;
14334                         /* Now what is our gain status? */
14335                         if (bbr->r_ctl.crte->rate < rate_wanted) {
14336                                 /* We have a problem */
14337                                 bbr_setup_less_of_rate(bbr, cts,
14338                                                        bbr->r_ctl.crte->rate, rate_wanted);
14339                         } else {
14340                                 /* We are good */
14341                                 bbr->gain_is_limited = 0;
14342                                 bbr->skip_gain = 0;
14343                         }
14344                         tcp_bbr_tso_size_check(bbr, cts);
14345                 } else {
14346                         bbr_type_log_hdwr_pacing(bbr,
14347                                                  inp->inp_route.ro_rt->rt_ifp,
14348                                                  rate_wanted,
14349                                                  0,
14350                                                  __LINE__, cts, err);
14351                         BBR_STAT_INC(bbr_hdwr_rl_add_fail);
14352                 }
14353         }
14354         if (bbr->bbr_hdrw_pacing) {
14355                 /*
14356                  * Worry about cases where the route
14357                  * changes or something happened that we
14358                  * lost our hardware pacing possibly during
14359                  * the last ip_output call.
14360                  */
14361                 if (inp->inp_snd_tag == NULL) {
14362                         /* A change during ip output disabled hw pacing? */
14363                         bbr->bbr_hdrw_pacing = 0;
14364                 } else if ((inp->inp_route.ro_rt == NULL) ||
14365                     (inp->inp_route.ro_rt->rt_ifp != inp->inp_snd_tag->ifp)) {
14366                         /*
14367                          * We had an interface or route change,
14368                          * detach from the current hdwr pacing
14369                          * and setup to re-attempt next go
14370                          * round.
14371                          */
14372                         bbr->bbr_hdrw_pacing = 0;
14373                         bbr->bbr_attempt_hdwr_pace = 0;
14374                         tcp_rel_pacing_rate(bbr->r_ctl.crte, bbr->rc_tp);
14375                         tcp_bbr_tso_size_check(bbr, cts);
14376                 }
14377         }
14378         /*
14379          * Data sent (as far as we can tell). If this advertises a larger
14380          * window than any other segment, then remember the size of the
14381          * advertised window. Any pending ACK has now been sent.
14382          */
14383         if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
14384                 tp->rcv_adv = tp->rcv_nxt + recwin;
14385
14386         tp->last_ack_sent = tp->rcv_nxt;
14387         if ((error == 0) &&
14388             (bbr->r_ctl.rc_pace_max_segs > tp->t_maxseg) &&
14389             (doing_tlp == 0) &&
14390             (tso == 0) &&
14391             (hw_tls == 0) &&
14392             (len > 0) &&
14393             ((flags & TH_RST) == 0) &&
14394             (IN_RECOVERY(tp->t_flags) == 0) &&
14395             (bbr->rc_in_persist == 0) &&
14396             ((tp->t_flags & TF_FORCEDATA) == 0) &&
14397             (tot_len < bbr->r_ctl.rc_pace_max_segs)) {
14398                 /*
14399                  * For non-tso we need to goto again until we have sent out
14400                  * enough data to match what we are hptsi out every hptsi
14401                  * interval.
14402                  */
14403                 if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
14404                         /* Make sure snd_nxt is drug up */
14405                         tp->snd_nxt = tp->snd_max;
14406                 }
14407                 if (rsm != NULL) {
14408                         rsm = NULL;
14409                         goto skip_again;
14410                 }
14411                 rsm = NULL;
14412                 sack_rxmit = 0;
14413                 tp->t_flags &= ~(TF_ACKNOW | TF_DELACK | TF_FORCEDATA);
14414                 goto again;
14415         }
14416 skip_again:
14417         if (((flags & (TH_RST | TH_SYN | TH_FIN)) == 0) && tot_len) {
14418                 /*
14419                  * Calculate/Re-Calculate the hptsi slot in usecs based on
14420                  * what we have sent so far
14421                  */
14422                 slot = bbr_get_pacing_delay(bbr, bbr->r_ctl.rc_bbr_hptsi_gain, tot_len, cts, 0);
14423                 if (bbr->rc_no_pacing)
14424                         slot = 0;
14425         }
14426         tp->t_flags &= ~(TF_ACKNOW | TF_DELACK | TF_FORCEDATA);
14427 enobufs:
14428         if (bbr->rc_use_google == 0)
14429                 bbr_check_bbr_for_state(bbr, cts, __LINE__, 0);
14430         bbr_cwnd_limiting(tp, bbr, ctf_flight_size(tp, (bbr->r_ctl.rc_sacked +
14431                                                         bbr->r_ctl.rc_lost_bytes)));
14432         bbr->rc_output_starts_timer = 1;
14433         if (bbr->bbr_use_rack_cheat &&
14434             (more_to_rxt ||
14435              ((bbr->r_ctl.rc_resend = bbr_check_recovery_mode(tp, bbr, cts)) != NULL))) {
14436                 /* Rack cheats and shotguns out all rxt's 1ms apart */
14437                 if (slot > 1000)
14438                         slot = 1000;
14439         }
14440         if (bbr->bbr_hdrw_pacing && (bbr->hw_pacing_set == 0)) {
14441                 /*
14442                  * We don't change the tso size until some number of sends
14443                  * to give the hardware commands time to get down
14444                  * to the interface.
14445                  */
14446                 bbr->r_ctl.bbr_hdwr_cnt_noset_snt++;
14447                 if (bbr->r_ctl.bbr_hdwr_cnt_noset_snt >= bbr_hdwr_pacing_delay_cnt) {
14448                         bbr->hw_pacing_set = 1;
14449                         tcp_bbr_tso_size_check(bbr, cts);
14450                 }
14451         }
14452         bbr_start_hpts_timer(bbr, tp, cts, 12, slot, tot_len);
14453         if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
14454                 /* Make sure snd_nxt is drug up */
14455                 tp->snd_nxt = tp->snd_max;
14456         }
14457         return (error);
14458
14459 }
14460
14461 /*
14462  * See bbr_output_wtime() for return values.
14463  */
14464 static int
14465 bbr_output(struct tcpcb *tp)
14466 {
14467         int32_t ret;
14468         struct timeval tv;
14469         struct tcp_bbr *bbr;
14470
14471         NET_EPOCH_ASSERT();
14472
14473         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14474         INP_WLOCK_ASSERT(tp->t_inpcb);
14475         (void)tcp_get_usecs(&tv);
14476         ret = bbr_output_wtime(tp, &tv);
14477         return (ret);
14478 }
14479
14480 static void
14481 bbr_mtu_chg(struct tcpcb *tp)
14482 {
14483         struct tcp_bbr *bbr;
14484         struct bbr_sendmap *rsm, *frsm = NULL;
14485         uint32_t maxseg;
14486
14487         /*
14488          * The MTU has changed. a) Clear the sack filter. b) Mark everything
14489          * over the current size as SACK_PASS so a retransmit will occur.
14490          */
14491
14492         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14493         maxseg = tp->t_maxseg - bbr->rc_last_options;
14494         sack_filter_clear(&bbr->r_ctl.bbr_sf, tp->snd_una);
14495         TAILQ_FOREACH(rsm, &bbr->r_ctl.rc_map, r_next) {
14496                 /* Don't mess with ones acked (by sack?) */
14497                 if (rsm->r_flags & BBR_ACKED)
14498                         continue;
14499                 if ((rsm->r_end - rsm->r_start) > maxseg) {
14500                         /*
14501                          * We mark sack-passed on all the previous large
14502                          * sends we did. This will force them to retransmit.
14503                          */
14504                         rsm->r_flags |= BBR_SACK_PASSED;
14505                         if (((rsm->r_flags & BBR_MARKED_LOST) == 0) &&
14506                             bbr_is_lost(bbr, rsm, bbr->r_ctl.rc_rcvtime)) {
14507                                 bbr->r_ctl.rc_lost_bytes += rsm->r_end - rsm->r_start;
14508                                 bbr->r_ctl.rc_lost += rsm->r_end - rsm->r_start;
14509                                 rsm->r_flags |= BBR_MARKED_LOST;
14510                         }
14511                         if (frsm == NULL)
14512                                 frsm = rsm;
14513                 }
14514         }
14515         if (frsm) {
14516                 bbr->r_ctl.rc_resend = frsm;
14517         }
14518 }
14519
14520 /*
14521  * bbr_ctloutput() must drop the inpcb lock before performing copyin on
14522  * socket option arguments.  When it re-acquires the lock after the copy, it
14523  * has to revalidate that the connection is still valid for the socket
14524  * option.
14525  */
14526 static int
14527 bbr_set_sockopt(struct socket *so, struct sockopt *sopt,
14528                 struct inpcb *inp, struct tcpcb *tp, struct tcp_bbr *bbr)
14529 {
14530         int32_t error = 0, optval;
14531
14532         switch (sopt->sopt_name) {
14533         case TCP_RACK_PACE_MAX_SEG:
14534         case TCP_RACK_MIN_TO:
14535         case TCP_RACK_REORD_THRESH:
14536         case TCP_RACK_REORD_FADE:
14537         case TCP_RACK_TLP_THRESH:
14538         case TCP_RACK_PKT_DELAY:
14539         case TCP_BBR_ALGORITHM:
14540         case TCP_BBR_TSLIMITS:
14541         case TCP_BBR_IWINTSO:
14542         case TCP_BBR_RECFORCE:
14543         case TCP_BBR_STARTUP_PG:
14544         case TCP_BBR_DRAIN_PG:
14545         case TCP_BBR_RWND_IS_APP:
14546         case TCP_BBR_PROBE_RTT_INT:
14547         case TCP_BBR_PROBE_RTT_GAIN:
14548         case TCP_BBR_PROBE_RTT_LEN:
14549         case TCP_BBR_STARTUP_LOSS_EXIT:
14550         case TCP_BBR_USEDEL_RATE:
14551         case TCP_BBR_MIN_RTO:
14552         case TCP_BBR_MAX_RTO:
14553         case TCP_BBR_PACE_PER_SEC:
14554         case TCP_DELACK:
14555         case TCP_BBR_PACE_DEL_TAR:
14556         case TCP_BBR_SEND_IWND_IN_TSO:
14557         case TCP_BBR_EXTRA_STATE:
14558         case TCP_BBR_UTTER_MAX_TSO:
14559         case TCP_BBR_MIN_TOPACEOUT:
14560         case TCP_BBR_FLOOR_MIN_TSO:
14561         case TCP_BBR_TSTMP_RAISES:
14562         case TCP_BBR_POLICER_DETECT:
14563         case TCP_BBR_USE_RACK_CHEAT:
14564         case TCP_DATA_AFTER_CLOSE:
14565         case TCP_BBR_HDWR_PACE:
14566         case TCP_BBR_PACE_SEG_MAX:
14567         case TCP_BBR_PACE_SEG_MIN:
14568         case TCP_BBR_PACE_CROSS:
14569         case TCP_BBR_PACE_OH:
14570 #ifdef NETFLIX_PEAKRATE
14571         case TCP_MAXPEAKRATE:
14572 #endif
14573         case TCP_BBR_TMR_PACE_OH:
14574         case TCP_BBR_RACK_RTT_USE:
14575         case TCP_BBR_RETRAN_WTSO:
14576                 break;
14577         default:
14578                 return (tcp_default_ctloutput(so, sopt, inp, tp));
14579                 break;
14580         }
14581         INP_WUNLOCK(inp);
14582         error = sooptcopyin(sopt, &optval, sizeof(optval), sizeof(optval));
14583         if (error)
14584                 return (error);
14585         INP_WLOCK(inp);
14586         if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
14587                 INP_WUNLOCK(inp);
14588                 return (ECONNRESET);
14589         }
14590         tp = intotcpcb(inp);
14591         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
14592         switch (sopt->sopt_name) {
14593         case TCP_BBR_PACE_PER_SEC:
14594                 BBR_OPTS_INC(tcp_bbr_pace_per_sec);
14595                 bbr->r_ctl.bbr_hptsi_per_second = optval;
14596                 break;
14597         case TCP_BBR_PACE_DEL_TAR:
14598                 BBR_OPTS_INC(tcp_bbr_pace_del_tar);
14599                 bbr->r_ctl.bbr_hptsi_segments_delay_tar = optval;
14600                 break;
14601         case TCP_BBR_PACE_SEG_MAX:
14602                 BBR_OPTS_INC(tcp_bbr_pace_seg_max);
14603                 bbr->r_ctl.bbr_hptsi_segments_max = optval;
14604                 break;
14605         case TCP_BBR_PACE_SEG_MIN:
14606                 BBR_OPTS_INC(tcp_bbr_pace_seg_min);
14607                 bbr->r_ctl.bbr_hptsi_bytes_min = optval;
14608                 break;
14609         case TCP_BBR_PACE_CROSS:
14610                 BBR_OPTS_INC(tcp_bbr_pace_cross);
14611                 bbr->r_ctl.bbr_cross_over = optval;
14612                 break;
14613         case TCP_BBR_ALGORITHM:
14614                 BBR_OPTS_INC(tcp_bbr_algorithm);
14615                 if (optval && (bbr->rc_use_google == 0)) {
14616                         /* Turn on the google mode */
14617                         bbr_google_mode_on(bbr);
14618                         if ((optval > 3) && (optval < 500)) {
14619                                 /*
14620                                  * Must be at least greater than .3%
14621                                  * and must be less than 50.0%.
14622                                  */
14623                                 bbr->r_ctl.bbr_google_discount = optval;
14624                         }
14625                 } else if ((optval == 0) && (bbr->rc_use_google == 1)) {
14626                         /* Turn off the google mode */
14627                         bbr_google_mode_off(bbr);
14628                 }
14629                 break;
14630         case TCP_BBR_TSLIMITS:
14631                 BBR_OPTS_INC(tcp_bbr_tslimits);
14632                 if (optval == 1)
14633                         bbr->rc_use_ts_limit = 1;
14634                 else if (optval == 0)
14635                         bbr->rc_use_ts_limit = 0;
14636                 else
14637                         error = EINVAL;
14638                 break;
14639
14640         case TCP_BBR_IWINTSO:
14641                 BBR_OPTS_INC(tcp_bbr_iwintso);
14642                 if ((optval >= 0) && (optval < 128)) {
14643                         uint32_t twin;
14644
14645                         bbr->rc_init_win = optval;
14646                         twin = bbr_initial_cwnd(bbr, tp);
14647                         if ((bbr->rc_past_init_win == 0) && (twin > tp->snd_cwnd))
14648                                 tp->snd_cwnd = twin;
14649                         else
14650                                 error = EBUSY;
14651                 } else
14652                         error = EINVAL;
14653                 break;
14654         case TCP_BBR_STARTUP_PG:
14655                 BBR_OPTS_INC(tcp_bbr_startup_pg);
14656                 if ((optval > 0) && (optval < BBR_MAX_GAIN_VALUE)) {
14657                         bbr->r_ctl.rc_startup_pg = optval;
14658                         if (bbr->rc_bbr_state == BBR_STATE_STARTUP) {
14659                                 bbr->r_ctl.rc_bbr_hptsi_gain = optval;
14660                         }
14661                 } else
14662                         error = EINVAL;
14663                 break;
14664         case TCP_BBR_DRAIN_PG:
14665                 BBR_OPTS_INC(tcp_bbr_drain_pg);
14666                 if ((optval > 0) && (optval < BBR_MAX_GAIN_VALUE))
14667                         bbr->r_ctl.rc_drain_pg = optval;
14668                 else
14669                         error = EINVAL;
14670                 break;
14671         case TCP_BBR_PROBE_RTT_LEN:
14672                 BBR_OPTS_INC(tcp_bbr_probertt_len);
14673                 if (optval <= 1)
14674                         reset_time_small(&bbr->r_ctl.rc_rttprop, (optval * USECS_IN_SECOND));
14675                 else
14676                         error = EINVAL;
14677                 break;
14678         case TCP_BBR_PROBE_RTT_GAIN:
14679                 BBR_OPTS_INC(tcp_bbr_probertt_gain);
14680                 if (optval <= BBR_UNIT)
14681                         bbr->r_ctl.bbr_rttprobe_gain_val = optval;
14682                 else
14683                         error = EINVAL;
14684                 break;
14685         case TCP_BBR_PROBE_RTT_INT:
14686                 BBR_OPTS_INC(tcp_bbr_probe_rtt_int);
14687                 if (optval > 1000)
14688                         bbr->r_ctl.rc_probertt_int = optval;
14689                 else
14690                         error = EINVAL;
14691                 break;
14692         case TCP_BBR_MIN_TOPACEOUT:
14693                 BBR_OPTS_INC(tcp_bbr_topaceout);
14694                 if (optval == 0) {
14695                         bbr->no_pacing_until = 0;
14696                         bbr->rc_no_pacing = 0;
14697                 } else if (optval <= 0x00ff) {
14698                         bbr->no_pacing_until = optval;
14699                         if ((bbr->r_ctl.rc_pkt_epoch < bbr->no_pacing_until) &&
14700                             (bbr->rc_bbr_state == BBR_STATE_STARTUP)){
14701                                 /* Turn on no pacing */
14702                                 bbr->rc_no_pacing = 1;
14703                         }
14704                 } else
14705                         error = EINVAL;
14706                 break;
14707         case TCP_BBR_STARTUP_LOSS_EXIT:
14708                 BBR_OPTS_INC(tcp_bbr_startup_loss_exit);
14709                 bbr->rc_loss_exit = optval;
14710                 break;
14711         case TCP_BBR_USEDEL_RATE:
14712                 error = EINVAL;
14713                 break;
14714         case TCP_BBR_MIN_RTO:
14715                 BBR_OPTS_INC(tcp_bbr_min_rto);
14716                 bbr->r_ctl.rc_min_rto_ms = optval;
14717                 break;
14718         case TCP_BBR_MAX_RTO:
14719                 BBR_OPTS_INC(tcp_bbr_max_rto);
14720                 bbr->rc_max_rto_sec = optval;
14721                 break;
14722         case TCP_RACK_MIN_TO:
14723                 /* Minimum time between rack t-o's in ms */
14724                 BBR_OPTS_INC(tcp_rack_min_to);
14725                 bbr->r_ctl.rc_min_to = optval;
14726                 break;
14727         case TCP_RACK_REORD_THRESH:
14728                 /* RACK reorder threshold (shift amount) */
14729                 BBR_OPTS_INC(tcp_rack_reord_thresh);
14730                 if ((optval > 0) && (optval < 31))
14731                         bbr->r_ctl.rc_reorder_shift = optval;
14732                 else
14733                         error = EINVAL;
14734                 break;
14735         case TCP_RACK_REORD_FADE:
14736                 /* Does reordering fade after ms time */
14737                 BBR_OPTS_INC(tcp_rack_reord_fade);
14738                 bbr->r_ctl.rc_reorder_fade = optval;
14739                 break;
14740         case TCP_RACK_TLP_THRESH:
14741                 /* RACK TLP theshold i.e. srtt+(srtt/N) */
14742                 BBR_OPTS_INC(tcp_rack_tlp_thresh);
14743                 if (optval)
14744                         bbr->rc_tlp_threshold = optval;
14745                 else
14746                         error = EINVAL;
14747                 break;
14748         case TCP_BBR_USE_RACK_CHEAT:
14749                 BBR_OPTS_INC(tcp_use_rackcheat);
14750                 if (bbr->rc_use_google) {
14751                         error = EINVAL;
14752                         break;
14753                 }
14754                 BBR_OPTS_INC(tcp_rack_cheat);
14755                 if (optval)
14756                         bbr->bbr_use_rack_cheat = 1;
14757                 else
14758                         bbr->bbr_use_rack_cheat = 0;
14759                 break;
14760         case TCP_BBR_FLOOR_MIN_TSO:
14761                 BBR_OPTS_INC(tcp_utter_max_tso);
14762                 if ((optval >= 0) && (optval < 40))
14763                         bbr->r_ctl.bbr_hptsi_segments_floor = optval;
14764                 else
14765                         error = EINVAL;
14766                 break;
14767         case TCP_BBR_UTTER_MAX_TSO:
14768                 BBR_OPTS_INC(tcp_utter_max_tso);
14769                 if ((optval >= 0) && (optval < 0xffff))
14770                         bbr->r_ctl.bbr_utter_max = optval;
14771                 else
14772                         error = EINVAL;
14773                 break;
14774
14775         case TCP_BBR_EXTRA_STATE:
14776                 BBR_OPTS_INC(tcp_extra_state);
14777                 if (optval)
14778                         bbr->rc_use_idle_restart = 1;
14779                 else
14780                         bbr->rc_use_idle_restart = 0;
14781                 break;
14782         case TCP_BBR_SEND_IWND_IN_TSO:
14783                 BBR_OPTS_INC(tcp_iwnd_tso);
14784                 if (optval) {
14785                         bbr->bbr_init_win_cheat = 1;
14786                         if (bbr->rc_past_init_win == 0) {
14787                                 uint32_t cts;
14788                                 cts = tcp_get_usecs(&bbr->rc_tv);
14789                                 tcp_bbr_tso_size_check(bbr, cts);
14790                         }
14791                 } else
14792                         bbr->bbr_init_win_cheat = 0;
14793                 break;
14794         case TCP_BBR_HDWR_PACE:
14795                 BBR_OPTS_INC(tcp_hdwr_pacing);
14796                 if (optval){
14797                         bbr->bbr_hdw_pace_ena = 1;
14798                         bbr->bbr_attempt_hdwr_pace = 0;
14799                 } else {
14800                         bbr->bbr_hdw_pace_ena = 0;
14801 #ifdef RATELIMIT
14802                         if (bbr->bbr_hdrw_pacing) {
14803                                 bbr->bbr_hdrw_pacing = 0;
14804                                 in_pcbdetach_txrtlmt(bbr->rc_inp);
14805                         }
14806 #endif
14807                 }
14808                 break;
14809
14810         case TCP_DELACK:
14811                 BBR_OPTS_INC(tcp_delack);
14812                 if (optval < 100) {
14813                         if (optval == 0) /* off */
14814                                 tp->t_delayed_ack = 0;
14815                         else if (optval == 1) /* on which is 2 */
14816                                 tp->t_delayed_ack = 2;
14817                         else /* higher than 2 and less than 100 */
14818                                 tp->t_delayed_ack = optval;
14819                         if (tp->t_flags & TF_DELACK) {
14820                                 tp->t_flags &= ~TF_DELACK;
14821                                 tp->t_flags |= TF_ACKNOW;
14822                                 bbr_output(tp);
14823                         }
14824                 } else
14825                         error = EINVAL;
14826                 break;
14827         case TCP_RACK_PKT_DELAY:
14828                 /* RACK added ms i.e. rack-rtt + reord + N */
14829                 BBR_OPTS_INC(tcp_rack_pkt_delay);
14830                 bbr->r_ctl.rc_pkt_delay = optval;
14831                 break;
14832 #ifdef NETFLIX_PEAKRATE
14833         case TCP_MAXPEAKRATE:
14834                 BBR_OPTS_INC(tcp_maxpeak);
14835                 error = tcp_set_maxpeakrate(tp, optval);
14836                 if (!error)
14837                         tp->t_peakrate_thr = tp->t_maxpeakrate;
14838                 break;
14839 #endif
14840         case TCP_BBR_RETRAN_WTSO:
14841                 BBR_OPTS_INC(tcp_retran_wtso);
14842                 if (optval)
14843                         bbr->rc_resends_use_tso = 1;
14844                 else
14845                         bbr->rc_resends_use_tso = 0;
14846                 break;
14847         case TCP_DATA_AFTER_CLOSE:
14848                 BBR_OPTS_INC(tcp_data_ac);
14849                 if (optval)
14850                         bbr->rc_allow_data_af_clo = 1;
14851                 else
14852                         bbr->rc_allow_data_af_clo = 0;
14853                 break;
14854         case TCP_BBR_POLICER_DETECT:
14855                 BBR_OPTS_INC(tcp_policer_det);
14856                 if (bbr->rc_use_google == 0)
14857                         error = EINVAL;
14858                 else if (optval)
14859                         bbr->r_use_policer = 1;
14860                 else
14861                         bbr->r_use_policer = 0;
14862                 break;
14863
14864         case TCP_BBR_TSTMP_RAISES:
14865                 BBR_OPTS_INC(tcp_ts_raises);
14866                 if (optval)
14867                         bbr->ts_can_raise = 1;
14868                 else
14869                         bbr->ts_can_raise = 0;
14870                 break;
14871         case TCP_BBR_TMR_PACE_OH:
14872                 BBR_OPTS_INC(tcp_pacing_oh_tmr);
14873                 if (bbr->rc_use_google) {
14874                         error = EINVAL;
14875                 } else {
14876                         if (optval)
14877                                 bbr->r_ctl.rc_incr_tmrs = 1;
14878                         else
14879                                 bbr->r_ctl.rc_incr_tmrs = 0;
14880                 }
14881                 break;
14882         case TCP_BBR_PACE_OH:
14883                 BBR_OPTS_INC(tcp_pacing_oh);
14884                 if (bbr->rc_use_google) {
14885                         error = EINVAL;
14886                 } else {
14887                         if (optval > (BBR_INCL_TCP_OH|
14888                                       BBR_INCL_IP_OH|
14889                                       BBR_INCL_ENET_OH)) {
14890                                 error = EINVAL;
14891                                 break;
14892                         }
14893                         if (optval & BBR_INCL_TCP_OH)
14894                                 bbr->r_ctl.rc_inc_tcp_oh = 1;
14895                         else
14896                                 bbr->r_ctl.rc_inc_tcp_oh = 0;
14897                         if (optval & BBR_INCL_IP_OH)
14898                                 bbr->r_ctl.rc_inc_ip_oh = 1;
14899                         else
14900                                 bbr->r_ctl.rc_inc_ip_oh = 0;
14901                         if (optval & BBR_INCL_ENET_OH)
14902                                 bbr->r_ctl.rc_inc_enet_oh = 1;
14903                         else
14904                                 bbr->r_ctl.rc_inc_enet_oh = 0;
14905                 }
14906                 break;
14907         default:
14908                 return (tcp_default_ctloutput(so, sopt, inp, tp));
14909                 break;
14910         }
14911 #ifdef NETFLIX_STATS
14912         tcp_log_socket_option(tp, sopt->sopt_name, optval, error);
14913 #endif
14914         INP_WUNLOCK(inp);
14915         return (error);
14916 }
14917
14918 /*
14919  * return 0 on success, error-num on failure
14920  */
14921 static int
14922 bbr_get_sockopt(struct socket *so, struct sockopt *sopt,
14923     struct inpcb *inp, struct tcpcb *tp, struct tcp_bbr *bbr)
14924 {
14925         int32_t error, optval;
14926
14927         /*
14928          * Because all our options are either boolean or an int, we can just
14929          * pull everything into optval and then unlock and copy. If we ever
14930          * add a option that is not a int, then this will have quite an
14931          * impact to this routine.
14932          */
14933         switch (sopt->sopt_name) {
14934         case TCP_BBR_PACE_PER_SEC:
14935                 optval = bbr->r_ctl.bbr_hptsi_per_second;
14936                 break;
14937         case TCP_BBR_PACE_DEL_TAR:
14938                 optval = bbr->r_ctl.bbr_hptsi_segments_delay_tar;
14939                 break;
14940         case TCP_BBR_PACE_SEG_MAX:
14941                 optval = bbr->r_ctl.bbr_hptsi_segments_max;
14942                 break;
14943         case TCP_BBR_MIN_TOPACEOUT:
14944                 optval = bbr->no_pacing_until;
14945                 break;
14946         case TCP_BBR_PACE_SEG_MIN:
14947                 optval = bbr->r_ctl.bbr_hptsi_bytes_min;
14948                 break;
14949         case TCP_BBR_PACE_CROSS:
14950                 optval = bbr->r_ctl.bbr_cross_over;
14951                 break;
14952         case TCP_BBR_ALGORITHM:
14953                 optval = bbr->rc_use_google;
14954                 break;
14955         case TCP_BBR_TSLIMITS:
14956                 optval = bbr->rc_use_ts_limit;
14957                 break;
14958         case TCP_BBR_IWINTSO:
14959                 optval = bbr->rc_init_win;
14960                 break;
14961         case TCP_BBR_STARTUP_PG:
14962                 optval = bbr->r_ctl.rc_startup_pg;
14963                 break;
14964         case TCP_BBR_DRAIN_PG:
14965                 optval = bbr->r_ctl.rc_drain_pg;
14966                 break;
14967         case TCP_BBR_PROBE_RTT_INT:
14968                 optval = bbr->r_ctl.rc_probertt_int;
14969                 break;
14970         case TCP_BBR_PROBE_RTT_LEN:
14971                 optval = (bbr->r_ctl.rc_rttprop.cur_time_limit / USECS_IN_SECOND);
14972                 break;
14973         case TCP_BBR_PROBE_RTT_GAIN:
14974                 optval = bbr->r_ctl.bbr_rttprobe_gain_val;
14975                 break;
14976         case TCP_BBR_STARTUP_LOSS_EXIT:
14977                 optval = bbr->rc_loss_exit;
14978                 break;
14979         case TCP_BBR_USEDEL_RATE:
14980                 error = EINVAL;
14981                 break;
14982         case TCP_BBR_MIN_RTO:
14983                 optval = bbr->r_ctl.rc_min_rto_ms;
14984                 break;
14985         case TCP_BBR_MAX_RTO:
14986                 optval = bbr->rc_max_rto_sec;
14987                 break;
14988         case TCP_RACK_PACE_MAX_SEG:
14989                 /* Max segments in a pace */
14990                 optval = bbr->r_ctl.rc_pace_max_segs;
14991                 break;
14992         case TCP_RACK_MIN_TO:
14993                 /* Minimum time between rack t-o's in ms */
14994                 optval = bbr->r_ctl.rc_min_to;
14995                 break;
14996         case TCP_RACK_REORD_THRESH:
14997                 /* RACK reorder threshold (shift amount) */
14998                 optval = bbr->r_ctl.rc_reorder_shift;
14999                 break;
15000         case TCP_RACK_REORD_FADE:
15001                 /* Does reordering fade after ms time */
15002                 optval = bbr->r_ctl.rc_reorder_fade;
15003                 break;
15004         case TCP_BBR_USE_RACK_CHEAT:
15005                 /* Do we use the rack cheat for rxt */
15006                 optval = bbr->bbr_use_rack_cheat;
15007                 break;
15008         case TCP_BBR_FLOOR_MIN_TSO:
15009                 optval = bbr->r_ctl.bbr_hptsi_segments_floor;
15010                 break;
15011         case TCP_BBR_UTTER_MAX_TSO:
15012                 optval = bbr->r_ctl.bbr_utter_max;
15013                 break;
15014         case TCP_BBR_SEND_IWND_IN_TSO:
15015                 /* Do we send TSO size segments initially */
15016                 optval = bbr->bbr_init_win_cheat;
15017                 break;
15018         case TCP_BBR_EXTRA_STATE:
15019                 optval = bbr->rc_use_idle_restart;
15020                 break;
15021         case TCP_RACK_TLP_THRESH:
15022                 /* RACK TLP theshold i.e. srtt+(srtt/N) */
15023                 optval = bbr->rc_tlp_threshold;
15024                 break;
15025         case TCP_RACK_PKT_DELAY:
15026                 /* RACK added ms i.e. rack-rtt + reord + N */
15027                 optval = bbr->r_ctl.rc_pkt_delay;
15028                 break;
15029         case TCP_BBR_RETRAN_WTSO:
15030                 optval = bbr->rc_resends_use_tso;
15031                 break;
15032         case TCP_DATA_AFTER_CLOSE:
15033                 optval = bbr->rc_allow_data_af_clo;
15034                 break;
15035         case TCP_DELACK:
15036                 optval = tp->t_delayed_ack;
15037                 break;
15038         case TCP_BBR_HDWR_PACE:
15039                 optval = bbr->bbr_hdw_pace_ena;
15040                 break;
15041         case TCP_BBR_POLICER_DETECT:
15042                 optval = bbr->r_use_policer;
15043                 break;
15044         case TCP_BBR_TSTMP_RAISES:
15045                 optval = bbr->ts_can_raise;
15046                 break;
15047         case TCP_BBR_TMR_PACE_OH:
15048                 optval = bbr->r_ctl.rc_incr_tmrs;
15049                 break;
15050         case TCP_BBR_PACE_OH:
15051                 optval = 0;
15052                 if (bbr->r_ctl.rc_inc_tcp_oh)
15053                         optval |= BBR_INCL_TCP_OH;
15054                 if (bbr->r_ctl.rc_inc_ip_oh)
15055                         optval |= BBR_INCL_IP_OH;
15056                 if (bbr->r_ctl.rc_inc_enet_oh)
15057                         optval |= BBR_INCL_ENET_OH;
15058                 break;
15059         default:
15060                 return (tcp_default_ctloutput(so, sopt, inp, tp));
15061                 break;
15062         }
15063         INP_WUNLOCK(inp);
15064         error = sooptcopyout(sopt, &optval, sizeof optval);
15065         return (error);
15066 }
15067
15068 /*
15069  * return 0 on success, error-num on failure
15070  */
15071 static int
15072 bbr_ctloutput(struct socket *so, struct sockopt *sopt, struct inpcb *inp, struct tcpcb *tp)
15073 {
15074         int32_t error = EINVAL;
15075         struct tcp_bbr *bbr;
15076
15077         bbr = (struct tcp_bbr *)tp->t_fb_ptr;
15078         if (bbr == NULL) {
15079                 /* Huh? */
15080                 goto out;
15081         }
15082         if (sopt->sopt_dir == SOPT_SET) {
15083                 return (bbr_set_sockopt(so, sopt, inp, tp, bbr));
15084         } else if (sopt->sopt_dir == SOPT_GET) {
15085                 return (bbr_get_sockopt(so, sopt, inp, tp, bbr));
15086         }
15087 out:
15088         INP_WUNLOCK(inp);
15089         return (error);
15090 }
15091
15092
15093 struct tcp_function_block __tcp_bbr = {
15094         .tfb_tcp_block_name = __XSTRING(STACKNAME),
15095         .tfb_tcp_output = bbr_output,
15096         .tfb_do_queued_segments = ctf_do_queued_segments,
15097         .tfb_do_segment_nounlock = bbr_do_segment_nounlock,
15098         .tfb_tcp_do_segment = bbr_do_segment,
15099         .tfb_tcp_ctloutput = bbr_ctloutput,
15100         .tfb_tcp_fb_init = bbr_init,
15101         .tfb_tcp_fb_fini = bbr_fini,
15102         .tfb_tcp_timer_stop_all = bbr_stopall,
15103         .tfb_tcp_timer_activate = bbr_timer_activate,
15104         .tfb_tcp_timer_active = bbr_timer_active,
15105         .tfb_tcp_timer_stop = bbr_timer_stop,
15106         .tfb_tcp_rexmit_tmr = bbr_remxt_tmr,
15107         .tfb_tcp_handoff_ok = bbr_handoff_ok,
15108         .tfb_tcp_mtu_chg = bbr_mtu_chg
15109 };
15110
15111 static const char *bbr_stack_names[] = {
15112         __XSTRING(STACKNAME),
15113 #ifdef STACKALIAS
15114         __XSTRING(STACKALIAS),
15115 #endif
15116 };
15117
15118 static bool bbr_mod_inited = false;
15119
15120 static int
15121 tcp_addbbr(module_t mod, int32_t type, void *data)
15122 {
15123         int32_t err = 0;
15124         int num_stacks;
15125
15126         switch (type) {
15127         case MOD_LOAD:
15128                 printf("Attempting to load " __XSTRING(MODNAME) "\n");
15129                 bbr_zone = uma_zcreate(__XSTRING(MODNAME) "_map",
15130                     sizeof(struct bbr_sendmap),
15131                     NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
15132                 bbr_pcb_zone = uma_zcreate(__XSTRING(MODNAME) "_pcb",
15133                     sizeof(struct tcp_bbr),
15134                     NULL, NULL, NULL, NULL, UMA_ALIGN_CACHE, 0);
15135                 sysctl_ctx_init(&bbr_sysctl_ctx);
15136                 bbr_sysctl_root = SYSCTL_ADD_NODE(&bbr_sysctl_ctx,
15137                     SYSCTL_STATIC_CHILDREN(_net_inet_tcp),
15138                     OID_AUTO,
15139 #ifdef STACKALIAS
15140                     __XSTRING(STACKALIAS),
15141 #else
15142                     __XSTRING(STACKNAME),
15143 #endif
15144                     CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
15145                     "");
15146                 if (bbr_sysctl_root == NULL) {
15147                         printf("Failed to add sysctl node\n");
15148                         err = EFAULT;
15149                         goto free_uma;
15150                 }
15151                 bbr_init_sysctls();
15152                 num_stacks = nitems(bbr_stack_names);
15153                 err = register_tcp_functions_as_names(&__tcp_bbr, M_WAITOK,
15154                     bbr_stack_names, &num_stacks);
15155                 if (err) {
15156                         printf("Failed to register %s stack name for "
15157                             "%s module\n", bbr_stack_names[num_stacks],
15158                             __XSTRING(MODNAME));
15159                         sysctl_ctx_free(&bbr_sysctl_ctx);
15160         free_uma:
15161                         uma_zdestroy(bbr_zone);
15162                         uma_zdestroy(bbr_pcb_zone);
15163                         bbr_counter_destroy();
15164                         printf("Failed to register " __XSTRING(MODNAME)
15165                             " module err:%d\n", err);
15166                         return (err);
15167                 }
15168                 tcp_lro_reg_mbufq();
15169                 bbr_mod_inited = true;
15170                 printf(__XSTRING(MODNAME) " is now available\n");
15171                 break;
15172         case MOD_QUIESCE:
15173                 err = deregister_tcp_functions(&__tcp_bbr, true, false);
15174                 break;
15175         case MOD_UNLOAD:
15176                 err = deregister_tcp_functions(&__tcp_bbr, false, true);
15177                 if (err == EBUSY)
15178                         break;
15179                 if (bbr_mod_inited) {
15180                         uma_zdestroy(bbr_zone);
15181                         uma_zdestroy(bbr_pcb_zone);
15182                         sysctl_ctx_free(&bbr_sysctl_ctx);
15183                         bbr_counter_destroy();
15184                         printf(__XSTRING(MODNAME)
15185                             " is now no longer available\n");
15186                         bbr_mod_inited = false;
15187                 }
15188                 tcp_lro_dereg_mbufq();
15189                 err = 0;
15190                 break;
15191         default:
15192                 return (EOPNOTSUPP);
15193         }
15194         return (err);
15195 }
15196
15197 static moduledata_t tcp_bbr = {
15198         .name = __XSTRING(MODNAME),
15199             .evhand = tcp_addbbr,
15200             .priv = 0
15201 };
15202
15203 MODULE_VERSION(MODNAME, 1);
15204 DECLARE_MODULE(MODNAME, tcp_bbr, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY);
15205 MODULE_DEPEND(MODNAME, tcphpts, 1, 1, 1);