sys/netinet/tcp_hpts.c

   1 /*-
   2  * Copyright (c) 2016-2018 Netflix Inc.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  */
  26 #include <sys/cdefs.h>
  27 __FBSDID("$FreeBSD$");
  28
  29 #include "opt_inet.h"
  30 #include "opt_inet6.h"
  31 #include "opt_ipsec.h"
  32 #include "opt_tcpdebug.h"
  33 /**
  34  * Some notes about usage.
  35  *
  36  * The tcp_hpts system is designed to provide a high precision timer
  37  * system for tcp. Its main purpose is to provide a mechanism for
  38  * pacing packets out onto the wire. It can be used in two ways
  39  * by a given TCP stack (and those two methods can be used simultaneously).
  40  *
  41  * First, and probably the main thing its used by Rack and BBR for, it can
  42  * be used to call tcp_output() of a transport stack at some time in the future.
  43  * The normal way this is done is that tcp_output() of the stack schedules
  44  * itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The
  45  * slot is the time from now that the stack wants to be called but it
  46  * must be converted to tcp_hpts's notion of slot. This is done with
  47  * one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical
  48  * call from the tcp_output() routine might look like:
  49  *
  50  * tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550));
  51  *
  52  * The above would schedule tcp_ouput() to be called in 550 useconds.
  53  * Note that if using this mechanism the stack will want to add near
  54  * its top a check to prevent unwanted calls (from user land or the
  55  * arrival of incoming ack's). So it would add something like:
  56  *
  57  * if (inp->inp_in_hpts)
  58  *    return;
  59  *
  60  * to prevent output processing until the time alotted has gone by.
  61  * Of course this is a bare bones example and the stack will probably
  62  * have more consideration then just the above.
  63  *
  64  * Now the tcp_hpts system will call tcp_output in one of two forms,
  65  * it will first check to see if the stack as defined a
  66  * tfb_tcp_output_wtime() function, if so that is the routine it
  67  * will call, if that function is not defined then it will call the
  68  * tfb_tcp_output() function. The only difference between these
  69  * two calls is that the former passes the time in to the function
  70  * so the function does not have to access the time (which tcp_hpts
  71  * already has). What these functions do is of course totally up
  72  * to the individual tcp stack.
  73  *
  74  * Now the second function (actually two functions I guess :D)
  75  * the tcp_hpts system provides is the  ability to either abort
  76  * a connection (later) or process  input on a connection.
  77  * Why would you want to do this? To keep processor locality.
  78  *
  79  * So in order to use the input redirection function the
  80  * stack changes its tcp_do_segment() routine to instead
  81  * of process the data call the function:
  82  *
  83  * tcp_queue_pkt_to_input()
  84  *
  85  * You will note that the arguments to this function look
  86  * a lot like tcp_do_segments's arguments. This function
  87  * will assure that the tcp_hpts system will
  88  * call the functions tfb_tcp_hpts_do_segment() from the
  89  * correct CPU. Note that multiple calls can get pushed
  90  * into the tcp_hpts system this will be indicated by
  91  * the next to last argument to tfb_tcp_hpts_do_segment()
  92  * (nxt_pkt). If nxt_pkt is a 1 then another packet is
  93  * coming. If nxt_pkt is a 0 then this is the last call
  94  * that the tcp_hpts system has available for the tcp stack.
  95  *
  96  * The other point of the input system is to be able to safely
  97  * drop a tcp connection without worrying about the recursive
  98  * locking that may be occuring on the INP_WLOCK. So if
  99  * a stack wants to drop a connection it calls:
 100  *
 101  *     tcp_set_inp_to_drop(tp, ETIMEDOUT)
 102  *
 103  * To schedule the tcp_hpts system to call
 104  *
 105  *    tcp_drop(tp, drop_reason)
 106  *
 107  * at a future point. This is quite handy to prevent locking
 108  * issues when dropping connections.
 109  *
 110  */
 111
 112 #include <sys/param.h>
 113 #include <sys/bus.h>
 114 #include <sys/interrupt.h>
 115 #include <sys/module.h>
 116 #include <sys/kernel.h>
 117 #include <sys/hhook.h>
 118 #include <sys/malloc.h>
 119 #include <sys/mbuf.h>
 120 #include <sys/proc.h>           /* for proc0 declaration */
 121 #include <sys/socket.h>
 122 #include <sys/socketvar.h>
 123 #include <sys/sysctl.h>
 124 #include <sys/systm.h>
 125 #include <sys/refcount.h>
 126 #include <sys/sched.h>
 127 #include <sys/queue.h>
 128 #include <sys/smp.h>
 129 #include <sys/counter.h>
 130 #include <sys/time.h>
 131 #include <sys/kthread.h>
 132 #include <sys/kern_prefetch.h>
 133
 134 #include <vm/uma.h>
 135
 136 #include <net/route.h>
 137 #include <net/vnet.h>
 138
 139 #define TCPSTATES               /* for logging */
 140
 141 #include <netinet/in.h>
 142 #include <netinet/in_kdtrace.h>
 143 #include <netinet/in_pcb.h>
 144 #include <netinet/ip.h>
 145 #include <netinet/ip_icmp.h>    /* required for icmp_var.h */
 146 #include <netinet/icmp_var.h>   /* for ICMP_BANDLIM */
 147 #include <netinet/ip_var.h>
 148 #include <netinet/ip6.h>
 149 #include <netinet6/in6_pcb.h>
 150 #include <netinet6/ip6_var.h>
 151 #define TCPOUTFLAGS
 152 #include <netinet/tcp.h>
 153 #include <netinet/tcp_fsm.h>
 154 #include <netinet/tcp_seq.h>
 155 #include <netinet/tcp_timer.h>
 156 #include <netinet/tcp_var.h>
 157 #include <netinet/tcpip.h>
 158 #include <netinet/cc/cc.h>
 159 #include <netinet/tcp_hpts.h>
 160
 161 #ifdef tcpdebug
 162 #include <netinet/tcp_debug.h>
 163 #endif                          /* tcpdebug */
 164 #ifdef tcp_offload
 165 #include <netinet/tcp_offload.h>
 166 #endif
 167
 168 #ifdef ipsec
 169 #include <netipsec/ipsec.h>
 170 #include <netipsec/ipsec6.h>
 171 #endif                          /* ipsec */
 172 #include "opt_rss.h"
 173
 174 MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts", "TCP hpts");
 175 #ifdef RSS
 176 static int tcp_bind_threads = 1;
 177 #else
 178 static int tcp_bind_threads = 0;
 179 #endif
 180 TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads);
 181
 182 static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG;
 183
 184 TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size);
 185
 186 static struct tcp_hptsi tcp_pace;
 187
 188 static int
 189 tcp_hptsi_lock_inpinfo(struct inpcb *inp,
 190     struct tcpcb **tp);
 191 static void tcp_wakehpts(struct tcp_hpts_entry *p);
 192 static void tcp_wakeinput(struct tcp_hpts_entry *p);
 193 static void tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv);
 194 static void tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick);
 195 static void tcp_hpts_thread(void *ctx);
 196 static void tcp_init_hptsi(void *st);
 197
 198 int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP;
 199 static int32_t tcp_hpts_callout_skip_swi = 0;
 200
 201 SYSCTL_DECL(_net_inet_tcp);
 202 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW, 0, "TCP Hpts controls");
 203
 204 #define timersub(tvp, uvp, vvp)                                         \
 205         do {                                                            \
 206                 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec;          \
 207                 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec;       \
 208                 if ((vvp)->tv_usec < 0) {                               \
 209                         (vvp)->tv_sec--;                                \
 210                         (vvp)->tv_usec += 1000000;                      \
 211                 }                                                       \
 212         } while (0)
 213
 214 static int32_t logging_on = 0;
 215 static int32_t hpts_sleep_max = (NUM_OF_HPTSI_SLOTS - 2);
 216 static int32_t tcp_hpts_precision = 120;
 217
 218 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW,
 219     &tcp_hpts_precision, 120,
 220     "Value for PRE() precision of callout");
 221
 222 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW,
 223     &logging_on, 0,
 224     "Turn on logging if compiled in");
 225
 226 counter_u64_t hpts_loops;
 227
 228 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, loops, CTLFLAG_RD,
 229     &hpts_loops, "Number of times hpts had to loop to catch up");
 230
 231 counter_u64_t back_tosleep;
 232
 233 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, no_tcbsfound, CTLFLAG_RD,
 234     &back_tosleep, "Number of times hpts found no tcbs");
 235
 236 static int32_t in_newts_every_tcb = 0;
 237
 238 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tsperpcb, CTLFLAG_RW,
 239     &in_newts_every_tcb, 0,
 240     "Do we have a new cts every tcb we process for input");
 241 static int32_t in_ts_percision = 0;
 242
 243 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tspercision, CTLFLAG_RW,
 244     &in_ts_percision, 0,
 245     "Do we use percise timestamp for clients on input");
 246 static int32_t out_newts_every_tcb = 0;
 247
 248 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tsperpcb, CTLFLAG_RW,
 249     &out_newts_every_tcb, 0,
 250     "Do we have a new cts every tcb we process for output");
 251 static int32_t out_ts_percision = 0;
 252
 253 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tspercision, CTLFLAG_RW,
 254     &out_ts_percision, 0,
 255     "Do we use a percise timestamp for every output cts");
 256
 257 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, maxsleep, CTLFLAG_RW,
 258     &hpts_sleep_max, 0,
 259     "The maximum time the hpts will sleep <1 - 254>");
 260
 261 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, minsleep, CTLFLAG_RW,
 262     &tcp_min_hptsi_time, 0,
 263     "The minimum time the hpts must sleep before processing more slots");
 264
 265 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, skip_swi, CTLFLAG_RW,
 266     &tcp_hpts_callout_skip_swi, 0,
 267     "Do we have the callout call directly to the hpts?");
 268
 269 static void
 270 __tcp_hpts_log_it(struct tcp_hpts_entry *hpts, struct inpcb *inp, int event, uint32_t slot,
 271     uint32_t ticknow, int32_t line)
 272 {
 273         struct hpts_log *pl;
 274
 275         HPTS_MTX_ASSERT(hpts);
 276         if (hpts->p_log == NULL)
 277                 return;
 278         pl = &hpts->p_log[hpts->p_log_at];
 279         hpts->p_log_at++;
 280         if (hpts->p_log_at >= hpts->p_logsize) {
 281                 hpts->p_log_at = 0;
 282                 hpts->p_log_wrapped = 1;
 283         }
 284         pl->inp = inp;
 285         if (inp) {
 286                 pl->t_paceslot = inp->inp_hptsslot;
 287                 pl->t_hptsreq = inp->inp_hpts_request;
 288                 pl->p_onhpts = inp->inp_in_hpts;
 289                 pl->p_oninput = inp->inp_in_input;
 290         } else {
 291                 pl->t_paceslot = 0;
 292                 pl->t_hptsreq = 0;
 293                 pl->p_onhpts = 0;
 294                 pl->p_oninput = 0;
 295         }
 296         pl->is_notempty = 1;
 297         pl->event = event;
 298         pl->line = line;
 299         pl->cts = tcp_get_usecs(NULL);
 300         pl->p_curtick = hpts->p_curtick;
 301         pl->p_prevtick = hpts->p_prevtick;
 302         pl->p_on_queue_cnt = hpts->p_on_queue_cnt;
 303         pl->ticknow = ticknow;
 304         pl->slot_req = slot;
 305         pl->p_nxt_slot = hpts->p_nxt_slot;
 306         pl->p_cur_slot = hpts->p_cur_slot;
 307         pl->p_hpts_sleep_time = hpts->p_hpts_sleep_time;
 308         pl->p_flags = (hpts->p_cpu & 0x7f);
 309         pl->p_flags <<= 7;
 310         pl->p_flags |= (hpts->p_num & 0x7f);
 311         pl->p_flags <<= 2;
 312         if (hpts->p_hpts_active) {
 313                 pl->p_flags |= HPTS_HPTS_ACTIVE;
 314         }
 315 }
 316
 317 #define tcp_hpts_log_it(a, b, c, d, e) __tcp_hpts_log_it(a, b, c, d, e, __LINE__)
 318
 319 static void
 320 hpts_timeout_swi(void *arg)
 321 {
 322         struct tcp_hpts_entry *hpts;
 323
 324         hpts = (struct tcp_hpts_entry *)arg;
 325         swi_sched(hpts->ie_cookie, 0);
 326 }
 327
 328 static void
 329 hpts_timeout_dir(void *arg)
 330 {
 331         tcp_hpts_thread(arg);
 332 }
 333
 334 static inline void
 335 hpts_sane_pace_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int clear)
 336 {
 337 #ifdef INVARIANTS
 338         if (mtx_owned(&hpts->p_mtx) == 0) {
 339                 /* We don't own the mutex? */
 340                 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
 341         }
 342         if (hpts->p_cpu != inp->inp_hpts_cpu) {
 343                 /* It is not the right cpu/mutex? */
 344                 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
 345         }
 346         if (inp->inp_in_hpts == 0) {
 347                 /* We are not on the hpts? */
 348                 panic("%s: hpts:%p inp:%p not on the hpts?", __FUNCTION__, hpts, inp);
 349         }
 350         if (TAILQ_EMPTY(head) &&
 351             (hpts->p_on_queue_cnt != 0)) {
 352                 /* We should not be empty with a queue count */
 353                 panic("%s hpts:%p hpts bucket empty but cnt:%d",
 354                     __FUNCTION__, hpts, hpts->p_on_queue_cnt);
 355         }
 356 #endif
 357         TAILQ_REMOVE(head, inp, inp_hpts);
 358         hpts->p_on_queue_cnt--;
 359         if (hpts->p_on_queue_cnt < 0) {
 360                 /* Count should not go negative .. */
 361 #ifdef INVARIANTS
 362                 panic("Hpts goes negative inp:%p hpts:%p",
 363                     inp, hpts);
 364 #endif
 365                 hpts->p_on_queue_cnt = 0;
 366         }
 367         if (clear) {
 368                 inp->inp_hpts_request = 0;
 369                 inp->inp_in_hpts = 0;
 370         }
 371 }
 372
 373 static inline void
 374 hpts_sane_pace_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int line, int noref)
 375 {
 376 #ifdef INVARIANTS
 377         if (mtx_owned(&hpts->p_mtx) == 0) {
 378                 /* We don't own the mutex? */
 379                 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
 380         }
 381         if (hpts->p_cpu != inp->inp_hpts_cpu) {
 382                 /* It is not the right cpu/mutex? */
 383                 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
 384         }
 385         if ((noref == 0) && (inp->inp_in_hpts == 1)) {
 386                 /* We are already on the hpts? */
 387                 panic("%s: hpts:%p inp:%p already on the hpts?", __FUNCTION__, hpts, inp);
 388         }
 389 #endif
 390         TAILQ_INSERT_TAIL(head, inp, inp_hpts);
 391         inp->inp_in_hpts = 1;
 392         hpts->p_on_queue_cnt++;
 393         if (noref == 0) {
 394                 in_pcbref(inp);
 395         }
 396 }
 397
 398 static inline void
 399 hpts_sane_input_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, int clear)
 400 {
 401 #ifdef INVARIANTS
 402         if (mtx_owned(&hpts->p_mtx) == 0) {
 403                 /* We don't own the mutex? */
 404                 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
 405         }
 406         if (hpts->p_cpu != inp->inp_input_cpu) {
 407                 /* It is not the right cpu/mutex? */
 408                 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
 409         }
 410         if (inp->inp_in_input == 0) {
 411                 /* We are not on the input hpts? */
 412                 panic("%s: hpts:%p inp:%p not on the input hpts?", __FUNCTION__, hpts, inp);
 413         }
 414 #endif
 415         TAILQ_REMOVE(&hpts->p_input, inp, inp_input);
 416         hpts->p_on_inqueue_cnt--;
 417         if (hpts->p_on_inqueue_cnt < 0) {
 418 #ifdef INVARIANTS
 419                 panic("Hpts in goes negative inp:%p hpts:%p",
 420                     inp, hpts);
 421 #endif
 422                 hpts->p_on_inqueue_cnt = 0;
 423         }
 424 #ifdef INVARIANTS
 425         if (TAILQ_EMPTY(&hpts->p_input) &&
 426             (hpts->p_on_inqueue_cnt != 0)) {
 427                 /* We should not be empty with a queue count */
 428                 panic("%s hpts:%p in_hpts input empty but cnt:%d",
 429                     __FUNCTION__, hpts, hpts->p_on_inqueue_cnt);
 430         }
 431 #endif
 432         if (clear)
 433                 inp->inp_in_input = 0;
 434 }
 435
 436 static inline void
 437 hpts_sane_input_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, int line)
 438 {
 439 #ifdef INVARIANTS
 440         if (mtx_owned(&hpts->p_mtx) == 0) {
 441                 /* We don't own the mutex? */
 442                 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
 443         }
 444         if (hpts->p_cpu != inp->inp_input_cpu) {
 445                 /* It is not the right cpu/mutex? */
 446                 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
 447         }
 448         if (inp->inp_in_input == 1) {
 449                 /* We are already on the input hpts? */
 450                 panic("%s: hpts:%p inp:%p already on the input hpts?", __FUNCTION__, hpts, inp);
 451         }
 452 #endif
 453         TAILQ_INSERT_TAIL(&hpts->p_input, inp, inp_input);
 454         inp->inp_in_input = 1;
 455         hpts->p_on_inqueue_cnt++;
 456         in_pcbref(inp);
 457 }
 458
 459 static int
 460 sysctl_tcp_hpts_log(SYSCTL_HANDLER_ARGS)
 461 {
 462         struct tcp_hpts_entry *hpts;
 463         size_t sz;
 464         int32_t logging_was, i;
 465         int32_t error = 0;
 466
 467         /*
 468          * HACK: Turn off logging so no locks are required this really needs
 469          * a memory barrier :)
 470          */
 471         logging_was = logging_on;
 472         logging_on = 0;
 473         if (!req->oldptr) {
 474                 /* How much? */
 475                 sz = 0;
 476                 for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
 477                         hpts = tcp_pace.rp_ent[i];
 478                         if (hpts->p_log == NULL)
 479                                 continue;
 480                         sz += (sizeof(struct hpts_log) * hpts->p_logsize);
 481                 }
 482                 error = SYSCTL_OUT(req, 0, sz);
 483         } else {
 484                 for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
 485                         hpts = tcp_pace.rp_ent[i];
 486                         if (hpts->p_log == NULL)
 487                                 continue;
 488                         if (hpts->p_log_wrapped)
 489                                 sz = (sizeof(struct hpts_log) * hpts->p_logsize);
 490                         else
 491                                 sz = (sizeof(struct hpts_log) * hpts->p_log_at);
 492                         error = SYSCTL_OUT(req, hpts->p_log, sz);
 493                 }
 494         }
 495         logging_on = logging_was;
 496         return error;
 497 }
 498
 499 SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
 500     0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log");
 501
 502
 503 /*
 504  * Try to get the INP_INFO lock.
 505  *
 506  * This function always succeeds in getting the lock. It will clear
 507  * *tpp and return (1) if something critical changed while the inpcb
 508  * was unlocked. Otherwise, it will leave *tpp unchanged and return (0).
 509  *
 510  * This function relies on the fact that the hpts always holds a
 511  * reference on the inpcb while the segment is on the hptsi wheel and
 512  * in the input queue.
 513  *
 514  */
 515 static int
 516 tcp_hptsi_lock_inpinfo(struct inpcb *inp, struct tcpcb **tpp)
 517 {
 518         struct tcp_function_block *tfb;
 519         struct tcpcb *tp;
 520         void *ptr;
 521
 522         /* Try the easy way. */
 523         if (INP_INFO_TRY_RLOCK(&V_tcbinfo))
 524                 return (0);
 525
 526         /*
 527          * OK, let's try the hard way. We'll save the function pointer block
 528          * to make sure that doesn't change while we aren't holding the
 529          * lock.
 530          */
 531         tp = *tpp;
 532         tfb = tp->t_fb;
 533         ptr = tp->t_fb_ptr;
 534         INP_WUNLOCK(inp);
 535         INP_INFO_RLOCK(&V_tcbinfo);
 536         INP_WLOCK(inp);
 537         /* If the session went away, return an error. */
 538         if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
 539             (inp->inp_flags2 & INP_FREED)) {
 540                 *tpp = NULL;
 541                 return (1);
 542         }
 543         /*
 544          * If the function block or stack-specific data block changed,
 545          * report an error.
 546          */
 547         tp = intotcpcb(inp);
 548         if ((tp->t_fb != tfb) && (tp->t_fb_ptr != ptr)) {
 549                 *tpp = NULL;
 550                 return (1);
 551         }
 552         return (0);
 553 }
 554
 555
 556 static void
 557 tcp_wakehpts(struct tcp_hpts_entry *hpts)
 558 {
 559         HPTS_MTX_ASSERT(hpts);
 560         swi_sched(hpts->ie_cookie, 0);
 561         if (hpts->p_hpts_active == 2) {
 562                 /* Rare sleeping on a ENOBUF */
 563                 wakeup_one(hpts);
 564         }
 565 }
 566
 567 static void
 568 tcp_wakeinput(struct tcp_hpts_entry *hpts)
 569 {
 570         HPTS_MTX_ASSERT(hpts);
 571         swi_sched(hpts->ie_cookie, 0);
 572         if (hpts->p_hpts_active == 2) {
 573                 /* Rare sleeping on a ENOBUF */
 574                 wakeup_one(hpts);
 575         }
 576 }
 577
 578 struct tcp_hpts_entry *
 579 tcp_cur_hpts(struct inpcb *inp)
 580 {
 581         int32_t hpts_num;
 582         struct tcp_hpts_entry *hpts;
 583
 584         hpts_num = inp->inp_hpts_cpu;
 585         hpts = tcp_pace.rp_ent[hpts_num];
 586         return (hpts);
 587 }
 588
 589 struct tcp_hpts_entry *
 590 tcp_hpts_lock(struct inpcb *inp)
 591 {
 592         struct tcp_hpts_entry *hpts;
 593         int32_t hpts_num;
 594
 595 again:
 596         hpts_num = inp->inp_hpts_cpu;
 597         hpts = tcp_pace.rp_ent[hpts_num];
 598 #ifdef INVARIANTS
 599         if (mtx_owned(&hpts->p_mtx)) {
 600                 panic("Hpts:%p owns mtx prior-to lock line:%d",
 601                     hpts, __LINE__);
 602         }
 603 #endif
 604         mtx_lock(&hpts->p_mtx);
 605         if (hpts_num != inp->inp_hpts_cpu) {
 606                 mtx_unlock(&hpts->p_mtx);
 607                 goto again;
 608         }
 609         return (hpts);
 610 }
 611
 612 struct tcp_hpts_entry *
 613 tcp_input_lock(struct inpcb *inp)
 614 {
 615         struct tcp_hpts_entry *hpts;
 616         int32_t hpts_num;
 617
 618 again:
 619         hpts_num = inp->inp_input_cpu;
 620         hpts = tcp_pace.rp_ent[hpts_num];
 621 #ifdef INVARIANTS
 622         if (mtx_owned(&hpts->p_mtx)) {
 623                 panic("Hpts:%p owns mtx prior-to lock line:%d",
 624                     hpts, __LINE__);
 625         }
 626 #endif
 627         mtx_lock(&hpts->p_mtx);
 628         if (hpts_num != inp->inp_input_cpu) {
 629                 mtx_unlock(&hpts->p_mtx);
 630                 goto again;
 631         }
 632         return (hpts);
 633 }
 634
 635 static void
 636 tcp_remove_hpts_ref(struct inpcb *inp, struct tcp_hpts_entry *hpts, int line)
 637 {
 638         int32_t add_freed;
 639
 640         if (inp->inp_flags2 & INP_FREED) {
 641                 /*
 642                  * Need to play a special trick so that in_pcbrele_wlocked
 643                  * does not return 1 when it really should have returned 0.
 644                  */
 645                 add_freed = 1;
 646                 inp->inp_flags2 &= ~INP_FREED;
 647         } else {
 648                 add_freed = 0;
 649         }
 650 #ifndef INP_REF_DEBUG
 651         if (in_pcbrele_wlocked(inp)) {
 652                 /*
 653                  * This should not happen. We have the inpcb referred to by
 654                  * the main socket (why we are called) and the hpts. It
 655                  * should always return 0.
 656                  */
 657                 panic("inpcb:%p release ret 1",
 658                     inp);
 659         }
 660 #else
 661         if (__in_pcbrele_wlocked(inp, line)) {
 662                 /*
 663                  * This should not happen. We have the inpcb referred to by
 664                  * the main socket (why we are called) and the hpts. It
 665                  * should always return 0.
 666                  */
 667                 panic("inpcb:%p release ret 1",
 668                     inp);
 669         }
 670 #endif
 671         if (add_freed) {
 672                 inp->inp_flags2 |= INP_FREED;
 673         }
 674 }
 675
 676 static void
 677 tcp_hpts_remove_locked_output(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line)
 678 {
 679         if (inp->inp_in_hpts) {
 680                 hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], 1);
 681                 tcp_remove_hpts_ref(inp, hpts, line);
 682         }
 683 }
 684
 685 static void
 686 tcp_hpts_remove_locked_input(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line)
 687 {
 688         HPTS_MTX_ASSERT(hpts);
 689         if (inp->inp_in_input) {
 690                 hpts_sane_input_remove(hpts, inp, 1);
 691                 tcp_remove_hpts_ref(inp, hpts, line);
 692         }
 693 }
 694
 695 /*
 696  * Called normally with the INP_LOCKED but it
 697  * does not matter, the hpts lock is the key
 698  * but the lock order allows us to hold the
 699  * INP lock and then get the hpts lock.
 700  *
 701  * Valid values in the flags are
 702  * HPTS_REMOVE_OUTPUT - remove from the output of the hpts.
 703  * HPTS_REMOVE_INPUT - remove from the input of the hpts.
 704  * Note that you can or both values together and get two
 705  * actions.
 706  */
 707 void
 708 __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line)
 709 {
 710         struct tcp_hpts_entry *hpts;
 711
 712         INP_WLOCK_ASSERT(inp);
 713         if (flags & HPTS_REMOVE_OUTPUT) {
 714                 hpts = tcp_hpts_lock(inp);
 715                 tcp_hpts_remove_locked_output(hpts, inp, flags, line);
 716                 mtx_unlock(&hpts->p_mtx);
 717         }
 718         if (flags & HPTS_REMOVE_INPUT) {
 719                 hpts = tcp_input_lock(inp);
 720                 tcp_hpts_remove_locked_input(hpts, inp, flags, line);
 721                 mtx_unlock(&hpts->p_mtx);
 722         }
 723 }
 724
 725 static inline int
 726 hpts_tick(struct tcp_hpts_entry *hpts, int32_t plus)
 727 {
 728         return ((hpts->p_prevtick + plus) % NUM_OF_HPTSI_SLOTS);
 729 }
 730
 731 static int
 732 tcp_queue_to_hpts_immediate_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line, int32_t noref)
 733 {
 734         int32_t need_wake = 0;
 735         uint32_t ticknow = 0;
 736
 737         HPTS_MTX_ASSERT(hpts);
 738         if (inp->inp_in_hpts == 0) {
 739                 /* Ok we need to set it on the hpts in the current slot */
 740                 if (hpts->p_hpts_active == 0) {
 741                         /* A sleeping hpts we want in next slot to run */
 742                         if (logging_on) {
 743                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, 0,
 744                                     hpts_tick(hpts, 1));
 745                         }
 746                         inp->inp_hptsslot = hpts_tick(hpts, 1);
 747                         inp->inp_hpts_request = 0;
 748                         if (logging_on) {
 749                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEP_BEFORE, 1, ticknow);
 750                         }
 751                         need_wake = 1;
 752                 } else if ((void *)inp == hpts->p_inp) {
 753                         /*
 754                          * We can't allow you to go into the same slot we
 755                          * are in. We must put you out.
 756                          */
 757                         inp->inp_hptsslot = hpts->p_nxt_slot;
 758                 } else
 759                         inp->inp_hptsslot = hpts->p_cur_slot;
 760                 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref);
 761                 inp->inp_hpts_request = 0;
 762                 if (logging_on) {
 763                         tcp_hpts_log_it(hpts, inp, HPTSLOG_IMMEDIATE, 0, 0);
 764                 }
 765                 if (need_wake) {
 766                         /*
 767                          * Activate the hpts if it is sleeping and its
 768                          * timeout is not 1.
 769                          */
 770                         if (logging_on) {
 771                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_HPTS, 0, ticknow);
 772                         }
 773                         hpts->p_direct_wake = 1;
 774                         tcp_wakehpts(hpts);
 775                 }
 776         }
 777         return (need_wake);
 778 }
 779
 780 int
 781 __tcp_queue_to_hpts_immediate(struct inpcb *inp, int32_t line)
 782 {
 783         int32_t ret;
 784         struct tcp_hpts_entry *hpts;
 785
 786         INP_WLOCK_ASSERT(inp);
 787         hpts = tcp_hpts_lock(inp);
 788         ret = tcp_queue_to_hpts_immediate_locked(inp, hpts, line, 0);
 789         mtx_unlock(&hpts->p_mtx);
 790         return (ret);
 791 }
 792
 793 static void
 794 tcp_hpts_insert_locked(struct tcp_hpts_entry *hpts, struct inpcb *inp, uint32_t slot, uint32_t cts, int32_t line,
 795     struct hpts_diag *diag, int32_t noref)
 796 {
 797         int32_t need_new_to = 0;
 798         int32_t need_wakeup = 0;
 799         uint32_t largest_slot;
 800         uint32_t ticknow = 0;
 801         uint32_t slot_calc;
 802
 803         HPTS_MTX_ASSERT(hpts);
 804         if (diag) {
 805                 memset(diag, 0, sizeof(struct hpts_diag));
 806                 diag->p_hpts_active = hpts->p_hpts_active;
 807                 diag->p_nxt_slot = hpts->p_nxt_slot;
 808                 diag->p_cur_slot = hpts->p_cur_slot;
 809                 diag->slot_req = slot;
 810         }
 811         if ((inp->inp_in_hpts == 0) || noref) {
 812                 inp->inp_hpts_request = slot;
 813                 if (slot == 0) {
 814                         /* Immediate */
 815                         tcp_queue_to_hpts_immediate_locked(inp, hpts, line, noref);
 816                         return;
 817                 }
 818                 if (hpts->p_hpts_active) {
 819                         /*
 820                          * Its slot - 1 since nxt_slot is the next tick that
 821                          * will go off since the hpts is awake
 822                          */
 823                         if (logging_on) {
 824                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_NORMAL, slot, 0);
 825                         }
 826                         /*
 827                          * We want to make sure that we don't place a inp in
 828                          * the range of p_cur_slot <-> p_nxt_slot. If we
 829                          * take from p_nxt_slot to the end, plus p_cur_slot
 830                          * and then take away 2, we will know how many is
 831                          * the max slots we can use.
 832                          */
 833                         if (hpts->p_nxt_slot > hpts->p_cur_slot) {
 834                                 /*
 835                                  * Non-wrap case nxt_slot <-> cur_slot we
 836                                  * don't want to land in. So the diff gives
 837                                  * us what is taken away from the number of
 838                                  * slots.
 839                                  */
 840                                 largest_slot = NUM_OF_HPTSI_SLOTS - (hpts->p_nxt_slot - hpts->p_cur_slot);
 841                         } else if (hpts->p_nxt_slot == hpts->p_cur_slot) {
 842                                 largest_slot = NUM_OF_HPTSI_SLOTS - 2;
 843                         } else {
 844                                 /*
 845                                  * Wrap case so the diff gives us the number
 846                                  * of slots that we can land in.
 847                                  */
 848                                 largest_slot = hpts->p_cur_slot - hpts->p_nxt_slot;
 849                         }
 850                         /*
 851                          * We take away two so we never have a problem (20
 852                          * usec's) out of 1024000 usecs
 853                          */
 854                         largest_slot -= 2;
 855                         if (inp->inp_hpts_request > largest_slot) {
 856                                 /*
 857                                  * Restrict max jump of slots and remember
 858                                  * leftover
 859                                  */
 860                                 slot = largest_slot;
 861                                 inp->inp_hpts_request -= largest_slot;
 862                         } else {
 863                                 /* This one will run when we hit it */
 864                                 inp->inp_hpts_request = 0;
 865                         }
 866                         if (hpts->p_nxt_slot == hpts->p_cur_slot)
 867                                 slot_calc = (hpts->p_nxt_slot + slot) % NUM_OF_HPTSI_SLOTS;
 868                         else
 869                                 slot_calc = (hpts->p_nxt_slot + slot - 1) % NUM_OF_HPTSI_SLOTS;
 870                         if (slot_calc == hpts->p_cur_slot) {
 871 #ifdef INVARIANTS
 872                                 /* TSNH */
 873                                 panic("Hpts:%p impossible slot calculation slot_calc:%u slot:%u largest:%u\n",
 874                                     hpts, slot_calc, slot, largest_slot);
 875 #endif
 876                                 if (slot_calc)
 877                                         slot_calc--;
 878                                 else
 879                                         slot_calc = NUM_OF_HPTSI_SLOTS - 1;
 880                         }
 881                         inp->inp_hptsslot = slot_calc;
 882                         if (diag) {
 883                                 diag->inp_hptsslot = inp->inp_hptsslot;
 884                         }
 885                 } else {
 886                         /*
 887                          * The hpts is sleeping, we need to figure out where
 888                          * it will wake up at and if we need to reschedule
 889                          * its time-out.
 890                          */
 891                         uint32_t have_slept, yet_to_sleep;
 892                         uint32_t slot_now;
 893                         struct timeval tv;
 894
 895                         ticknow = tcp_gethptstick(&tv);
 896                         slot_now = ticknow % NUM_OF_HPTSI_SLOTS;
 897                         /*
 898                          * The user wants to be inserted at (slot_now +
 899                          * slot) % NUM_OF_HPTSI_SLOTS, so lets set that up.
 900                          */
 901                         largest_slot = NUM_OF_HPTSI_SLOTS - 2;
 902                         if (inp->inp_hpts_request > largest_slot) {
 903                                 /* Adjust the residual in inp_hpts_request */
 904                                 slot = largest_slot;
 905                                 inp->inp_hpts_request -= largest_slot;
 906                         } else {
 907                                 /* No residual it all fits */
 908                                 inp->inp_hpts_request = 0;
 909                         }
 910                         inp->inp_hptsslot = (slot_now + slot) % NUM_OF_HPTSI_SLOTS;
 911                         if (diag) {
 912                                 diag->slot_now = slot_now;
 913                                 diag->inp_hptsslot = inp->inp_hptsslot;
 914                                 diag->p_on_min_sleep = hpts->p_on_min_sleep;
 915                         }
 916                         if (logging_on) {
 917                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, slot, ticknow);
 918                         }
 919                         /* Now do we need to restart the hpts's timer? */
 920                         if (TSTMP_GT(ticknow, hpts->p_curtick))
 921                                 have_slept = ticknow - hpts->p_curtick;
 922                         else
 923                                 have_slept = 0;
 924                         if (have_slept < hpts->p_hpts_sleep_time) {
 925                                 /* This should be what happens */
 926                                 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept;
 927                         } else {
 928                                 /* We are over-due */
 929                                 yet_to_sleep = 0;
 930                                 need_wakeup = 1;
 931                         }
 932                         if (diag) {
 933                                 diag->have_slept = have_slept;
 934                                 diag->yet_to_sleep = yet_to_sleep;
 935                                 diag->hpts_sleep_time = hpts->p_hpts_sleep_time;
 936                         }
 937                         if ((hpts->p_on_min_sleep == 0) && (yet_to_sleep > slot)) {
 938                                 /*
 939                                  * We need to reschedule the hptss time-out.
 940                                  */
 941                                 hpts->p_hpts_sleep_time = slot;
 942                                 need_new_to = slot * HPTS_TICKS_PER_USEC;
 943                         }
 944                 }
 945                 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref);
 946                 if (logging_on) {
 947                         tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERTED, slot, ticknow);
 948                 }
 949                 /*
 950                  * Now how far is the hpts sleeping to? if active is 1, its
 951                  * up and ticking we do nothing, otherwise we may need to
 952                  * reschedule its callout if need_new_to is set from above.
 953                  */
 954                 if (need_wakeup) {
 955                         if (logging_on) {
 956                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_RESCHEDULE, 1, 0);
 957                         }
 958                         hpts->p_direct_wake = 1;
 959                         tcp_wakehpts(hpts);
 960                         if (diag) {
 961                                 diag->need_new_to = 0;
 962                                 diag->co_ret = 0xffff0000;
 963                         }
 964                 } else if (need_new_to) {
 965                         int32_t co_ret;
 966                         struct timeval tv;
 967                         sbintime_t sb;
 968
 969                         tv.tv_sec = 0;
 970                         tv.tv_usec = 0;
 971                         while (need_new_to > HPTS_USEC_IN_SEC) {
 972                                 tv.tv_sec++;
 973                                 need_new_to -= HPTS_USEC_IN_SEC;
 974                         }
 975                         tv.tv_usec = need_new_to;
 976                         sb = tvtosbt(tv);
 977                         if (tcp_hpts_callout_skip_swi == 0) {
 978                                 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0,
 979                                     hpts_timeout_swi, hpts, hpts->p_cpu,
 980                                     (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
 981                         } else {
 982                                 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0,
 983                                     hpts_timeout_dir, hpts,
 984                                     hpts->p_cpu,
 985                                     C_PREL(tcp_hpts_precision));
 986                         }
 987                         if (diag) {
 988                                 diag->need_new_to = need_new_to;
 989                                 diag->co_ret = co_ret;
 990                         }
 991                 }
 992         } else {
 993 #ifdef INVARIANTS
 994                 panic("Hpts:%p tp:%p already on hpts and add?", hpts, inp);
 995 #endif
 996         }
 997 }
 998
 999 uint32_t
1000 tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag){
1001         struct tcp_hpts_entry *hpts;
1002         uint32_t slot_on, cts;
1003         struct timeval tv;
1004
1005         /*
1006          * We now return the next-slot the hpts will be on, beyond its
1007          * current run (if up) or where it was when it stopped if it is
1008          * sleeping.
1009          */
1010         INP_WLOCK_ASSERT(inp);
1011         hpts = tcp_hpts_lock(inp);
1012         if (in_ts_percision)
1013                 microuptime(&tv);
1014         else
1015                 getmicrouptime(&tv);
1016         cts = tcp_tv_to_usectick(&tv);
1017         tcp_hpts_insert_locked(hpts, inp, slot, cts, line, diag, 0);
1018         slot_on = hpts->p_nxt_slot;
1019         mtx_unlock(&hpts->p_mtx);
1020         return (slot_on);
1021 }
1022
1023 uint32_t
1024 __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line){
1025         return (tcp_hpts_insert_diag(inp, slot, line, NULL));
1026 }
1027
1028 int
1029 __tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line)
1030 {
1031         int32_t retval = 0;
1032
1033         HPTS_MTX_ASSERT(hpts);
1034         if (inp->inp_in_input == 0) {
1035                 /* Ok we need to set it on the hpts in the current slot */
1036                 hpts_sane_input_insert(hpts, inp, line);
1037                 retval = 1;
1038                 if (hpts->p_hpts_active == 0) {
1039                         /*
1040                          * Activate the hpts if it is sleeping.
1041                          */
1042                         if (logging_on) {
1043                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_INPUT, 0, 0);
1044                         }
1045                         retval = 2;
1046                         hpts->p_direct_wake = 1;
1047                         tcp_wakeinput(hpts);
1048                 }
1049         } else if (hpts->p_hpts_active == 0) {
1050                 retval = 4;
1051                 hpts->p_direct_wake = 1;
1052                 tcp_wakeinput(hpts);
1053         }
1054         return (retval);
1055 }
1056
1057 void
1058 tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
1059     int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked)
1060 {
1061         /* Setup packet for input first */
1062         INP_WLOCK_ASSERT(tp->t_inpcb);
1063         m->m_pkthdr.pace_thoff = (uint16_t) ((caddr_t)th - mtod(m, caddr_t));
1064         m->m_pkthdr.pace_tlen = (uint16_t) tlen;
1065         m->m_pkthdr.pace_drphdrlen = drop_hdrlen;
1066         m->m_pkthdr.pace_tos = iptos;
1067         m->m_pkthdr.pace_lock = (uint8_t) ti_locked;
1068         if (tp->t_in_pkt == NULL) {
1069                 tp->t_in_pkt = m;
1070                 tp->t_tail_pkt = m;
1071         } else {
1072                 tp->t_tail_pkt->m_nextpkt = m;
1073                 tp->t_tail_pkt = m;
1074         }
1075 }
1076
1077
1078 int32_t
1079 __tcp_queue_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
1080     int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, uint8_t ti_locked, int32_t line){
1081         struct tcp_hpts_entry *hpts;
1082         int32_t ret;
1083
1084         tcp_queue_pkt_to_input(tp, m, th, tlen, drop_hdrlen, iptos, ti_locked);
1085         hpts = tcp_input_lock(tp->t_inpcb);
1086         ret = __tcp_queue_to_input_locked(tp->t_inpcb, hpts, line);
1087         mtx_unlock(&hpts->p_mtx);
1088         return (ret);
1089 }
1090
1091 void
1092 __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line)
1093 {
1094         struct tcp_hpts_entry *hpts;
1095         struct tcpcb *tp;
1096
1097         tp = intotcpcb(inp);
1098         hpts = tcp_input_lock(tp->t_inpcb);
1099         if (inp->inp_in_input == 0) {
1100                 /* Ok we need to set it on the hpts in the current slot */
1101                 hpts_sane_input_insert(hpts, inp, line);
1102                 if (hpts->p_hpts_active == 0) {
1103                         /*
1104                          * Activate the hpts if it is sleeping.
1105                          */
1106                         hpts->p_direct_wake = 1;
1107                         tcp_wakeinput(hpts);
1108                 }
1109         } else if (hpts->p_hpts_active == 0) {
1110                 hpts->p_direct_wake = 1;
1111                 tcp_wakeinput(hpts);
1112         }
1113         inp->inp_hpts_drop_reas = reason;
1114         mtx_unlock(&hpts->p_mtx);
1115 }
1116
1117 static uint16_t
1118 hpts_random_cpu(struct inpcb *inp){
1119         /*
1120          * No flow type set distribute the load randomly.
1121          */
1122         uint16_t cpuid;
1123         uint32_t ran;
1124
1125         /*
1126          * If one has been set use it i.e. we want both in and out on the
1127          * same hpts.
1128          */
1129         if (inp->inp_input_cpu_set) {
1130                 return (inp->inp_input_cpu);
1131         } else if (inp->inp_hpts_cpu_set) {
1132                 return (inp->inp_hpts_cpu);
1133         }
1134         /* Nothing set use a random number */
1135         ran = arc4random();
1136         cpuid = (ran & 0xffff) % mp_ncpus;
1137         return (cpuid);
1138 }
1139
1140 static uint16_t
1141 hpts_cpuid(struct inpcb *inp){
1142         uint16_t cpuid;
1143
1144
1145         /*
1146          * If one has been set use it i.e. we want both in and out on the
1147          * same hpts.
1148          */
1149         if (inp->inp_input_cpu_set) {
1150                 return (inp->inp_input_cpu);
1151         } else if (inp->inp_hpts_cpu_set) {
1152                 return (inp->inp_hpts_cpu);
1153         }
1154         /* If one is set the other must be the same */
1155 #ifdef  RSS
1156         cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype);
1157         if (cpuid == NETISR_CPUID_NONE)
1158                 return (hpts_random_cpu(inp));
1159         else
1160                 return (cpuid);
1161 #else
1162         /*
1163          * We don't have a flowid -> cpuid mapping, so cheat and just map
1164          * unknown cpuids to curcpu.  Not the best, but apparently better
1165          * than defaulting to swi 0.
1166          */
1167         if (inp->inp_flowtype != M_HASHTYPE_NONE) {
1168                 cpuid = inp->inp_flowid % mp_ncpus;
1169                 return (cpuid);
1170         }
1171         cpuid = hpts_random_cpu(inp);
1172         return (cpuid);
1173 #endif
1174 }
1175
1176 /*
1177  * Do NOT try to optimize the processing of inp's
1178  * by first pulling off all the inp's into a temporary
1179  * list (e.g. TAILQ_CONCAT). If you do that the subtle
1180  * interactions of switching CPU's will kill because of
1181  * problems in the linked list manipulation. Basically
1182  * you would switch cpu's with the hpts mutex locked
1183  * but then while you were processing one of the inp's
1184  * some other one that you switch will get a new
1185  * packet on the different CPU. It will insert it
1186  * on the new hptss input list. Creating a temporary
1187  * link in the inp will not fix it either, since
1188  * the other hpts will be doing the same thing and
1189  * you will both end up using the temporary link.
1190  *
1191  * You will die in an ASSERT for tailq corruption if you
1192  * run INVARIANTS or you will die horribly without
1193  * INVARIANTS in some unknown way with a corrupt linked
1194  * list.
1195  */
1196 static void
1197 tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv)
1198 {
1199         struct mbuf *m, *n;
1200         struct tcpcb *tp;
1201         struct inpcb *inp;
1202         uint16_t drop_reason;
1203         int16_t set_cpu;
1204         uint32_t did_prefetch = 0;
1205         int32_t ti_locked = TI_UNLOCKED;
1206
1207         HPTS_MTX_ASSERT(hpts);
1208         while ((inp = TAILQ_FIRST(&hpts->p_input)) != NULL) {
1209                 HPTS_MTX_ASSERT(hpts);
1210                 hpts_sane_input_remove(hpts, inp, 0);
1211                 if (inp->inp_input_cpu_set == 0) {
1212                         set_cpu = 1;
1213                 } else {
1214                         set_cpu = 0;
1215                 }
1216                 hpts->p_inp = inp;
1217                 drop_reason = inp->inp_hpts_drop_reas;
1218                 inp->inp_in_input = 0;
1219                 mtx_unlock(&hpts->p_mtx);
1220                 if (drop_reason) {
1221                         INP_INFO_RLOCK(&V_tcbinfo);
1222                         ti_locked = TI_RLOCKED;
1223                 } else {
1224                         ti_locked = TI_UNLOCKED;
1225                 }
1226                 INP_WLOCK(inp);
1227                 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
1228                     (inp->inp_flags2 & INP_FREED)) {
1229 out:
1230                         hpts->p_inp = NULL;
1231                         if (ti_locked == TI_RLOCKED) {
1232                                 INP_INFO_RUNLOCK(&V_tcbinfo);
1233                         }
1234                         if (in_pcbrele_wlocked(inp) == 0) {
1235                                 INP_WUNLOCK(inp);
1236                         }
1237                         ti_locked = TI_UNLOCKED;
1238                         mtx_lock(&hpts->p_mtx);
1239                         continue;
1240                 }
1241                 tp = intotcpcb(inp);
1242                 if ((tp == NULL) || (tp->t_inpcb == NULL)) {
1243                         goto out;
1244                 }
1245                 if (drop_reason) {
1246                         /* This tcb is being destroyed for drop_reason */
1247                         m = tp->t_in_pkt;
1248                         if (m)
1249                                 n = m->m_nextpkt;
1250                         else
1251                                 n = NULL;
1252                         tp->t_in_pkt = NULL;
1253                         while (m) {
1254                                 m_freem(m);
1255                                 m = n;
1256                                 if (m)
1257                                         n = m->m_nextpkt;
1258                         }
1259                         tp = tcp_drop(tp, drop_reason);
1260                         INP_INFO_RUNLOCK(&V_tcbinfo);
1261                         if (tp == NULL) {
1262                                 INP_WLOCK(inp);
1263                         }
1264                         if (in_pcbrele_wlocked(inp) == 0)
1265                                 INP_WUNLOCK(inp);
1266                         mtx_lock(&hpts->p_mtx);
1267                         continue;
1268                 }
1269                 if (set_cpu) {
1270                         /*
1271                          * Setup so the next time we will move to the right
1272                          * CPU. This should be a rare event. It will
1273                          * sometimes happens when we are the client side
1274                          * (usually not the server). Somehow tcp_output()
1275                          * gets called before the tcp_do_segment() sets the
1276                          * intial state. This means the r_cpu and r_hpts_cpu
1277                          * is 0. We get on the hpts, and then tcp_input()
1278                          * gets called setting up the r_cpu to the correct
1279                          * value. The hpts goes off and sees the mis-match.
1280                          * We simply correct it here and the CPU will switch
1281                          * to the new hpts nextime the tcb gets added to the
1282                          * the hpts (not this time) :-)
1283                          */
1284                         tcp_set_hpts(inp);
1285                 }
1286                 CURVNET_SET(tp->t_vnet);
1287                 m = tp->t_in_pkt;
1288                 n = NULL;
1289                 if (m != NULL &&
1290                     (m->m_pkthdr.pace_lock == TI_RLOCKED ||
1291                     tp->t_state != TCPS_ESTABLISHED)) {
1292                         ti_locked = TI_RLOCKED;
1293                         if (tcp_hptsi_lock_inpinfo(inp, &tp)) {
1294                                 CURVNET_RESTORE();
1295                                 goto out;
1296                         }
1297                         m = tp->t_in_pkt;
1298                 }
1299                 if (in_newts_every_tcb) {
1300                         if (in_ts_percision)
1301                                 microuptime(tv);
1302                         else
1303                                 getmicrouptime(tv);
1304                 }
1305                 if (tp->t_fb_ptr != NULL) {
1306                         kern_prefetch(tp->t_fb_ptr, &did_prefetch);
1307                         did_prefetch = 1;
1308                 }
1309                 /* Any input work to do, if so do it first */
1310                 if ((m != NULL) && (m == tp->t_in_pkt)) {
1311                         struct tcphdr *th;
1312                         int32_t tlen, drop_hdrlen, nxt_pkt;
1313                         uint8_t iptos;
1314
1315                         n = m->m_nextpkt;
1316                         tp->t_in_pkt = tp->t_tail_pkt = NULL;
1317                         while (m) {
1318                                 th = (struct tcphdr *)(mtod(m, caddr_t)+m->m_pkthdr.pace_thoff);
1319                                 tlen = m->m_pkthdr.pace_tlen;
1320                                 drop_hdrlen = m->m_pkthdr.pace_drphdrlen;
1321                                 iptos = m->m_pkthdr.pace_tos;
1322                                 m->m_nextpkt = NULL;
1323                                 if (n)
1324                                         nxt_pkt = 1;
1325                                 else
1326                                         nxt_pkt = 0;
1327                                 inp->inp_input_calls = 1;
1328                                 if (tp->t_fb->tfb_tcp_hpts_do_segment) {
1329                                         /* Use the hpts specific do_segment */
1330                                         (*tp->t_fb->tfb_tcp_hpts_do_segment) (m, th, inp->inp_socket,
1331                                             tp, drop_hdrlen,
1332                                             tlen, iptos, ti_locked, nxt_pkt, tv);
1333                                 } else {
1334                                         /* Use the default do_segment */
1335                                         (*tp->t_fb->tfb_tcp_do_segment) (m, th, inp->inp_socket,
1336                                             tp, drop_hdrlen,
1337                                             tlen, iptos, ti_locked);
1338                                 }
1339                                 /*
1340                                  * Do segment returns unlocked we need the
1341                                  * lock again but we also need some kasserts
1342                                  * here.
1343                                  */
1344                                 INP_INFO_UNLOCK_ASSERT(&V_tcbinfo);
1345                                 INP_UNLOCK_ASSERT(inp);
1346                                 m = n;
1347                                 if (m)
1348                                         n = m->m_nextpkt;
1349                                 if (m != NULL &&
1350                                     m->m_pkthdr.pace_lock == TI_RLOCKED) {
1351                                         INP_INFO_RLOCK(&V_tcbinfo);
1352                                         ti_locked = TI_RLOCKED;
1353                                 } else
1354                                         ti_locked = TI_UNLOCKED;
1355                                 INP_WLOCK(inp);
1356                                 /*
1357                                  * Since we have an opening here we must
1358                                  * re-check if the tcb went away while we
1359                                  * were getting the lock(s).
1360                                  */
1361                                 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
1362                                     (inp->inp_flags2 & INP_FREED)) {
1363                         out_free:
1364                                         while (m) {
1365                                                 m_freem(m);
1366                                                 m = n;
1367                                                 if (m)
1368                                                         n = m->m_nextpkt;
1369                                         }
1370                                         CURVNET_RESTORE();
1371                                         goto out;
1372                                 }
1373                                 /*
1374                                  * Now that we hold the INP lock, check if
1375                                  * we need to upgrade our lock.
1376                                  */
1377                                 if (ti_locked == TI_UNLOCKED &&
1378                                     (tp->t_state != TCPS_ESTABLISHED)) {
1379                                         ti_locked = TI_RLOCKED;
1380                                         if (tcp_hptsi_lock_inpinfo(inp, &tp))
1381                                                 goto out_free;
1382                                 }
1383                         }       /** end while(m) */
1384                 }               /** end if ((m != NULL)  && (m == tp->t_in_pkt)) */
1385                 if (in_pcbrele_wlocked(inp) == 0)
1386                         INP_WUNLOCK(inp);
1387                 if (ti_locked == TI_RLOCKED)
1388                         INP_INFO_RUNLOCK(&V_tcbinfo);
1389                 INP_INFO_UNLOCK_ASSERT(&V_tcbinfo);
1390                 INP_UNLOCK_ASSERT(inp);
1391                 ti_locked = TI_UNLOCKED;
1392                 mtx_lock(&hpts->p_mtx);
1393                 hpts->p_inp = NULL;
1394                 CURVNET_RESTORE();
1395         }
1396 }
1397
1398 static int
1399 tcp_hpts_est_run(struct tcp_hpts_entry *hpts)
1400 {
1401         int32_t ticks_to_run;
1402
1403         if (hpts->p_prevtick && (SEQ_GT(hpts->p_curtick, hpts->p_prevtick))) {
1404                 ticks_to_run = hpts->p_curtick - hpts->p_prevtick;
1405                 if (ticks_to_run >= (NUM_OF_HPTSI_SLOTS - 1)) {
1406                         ticks_to_run = NUM_OF_HPTSI_SLOTS - 2;
1407                 }
1408         } else {
1409                 if (hpts->p_prevtick == hpts->p_curtick) {
1410                         /* This happens when we get woken up right away */
1411                         return (-1);
1412                 }
1413                 ticks_to_run = 1;
1414         }
1415         /* Set in where we will be when we catch up */
1416         hpts->p_nxt_slot = (hpts->p_cur_slot + ticks_to_run) % NUM_OF_HPTSI_SLOTS;
1417         if (hpts->p_nxt_slot == hpts->p_cur_slot) {
1418                 panic("Impossible math -- hpts:%p p_nxt_slot:%d p_cur_slot:%d ticks_to_run:%d",
1419                     hpts, hpts->p_nxt_slot, hpts->p_cur_slot, ticks_to_run);
1420         }
1421         return (ticks_to_run);
1422 }
1423
1424 static void
1425 tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick)
1426 {
1427         struct tcpcb *tp;
1428         struct inpcb *inp = NULL, *ninp;
1429         struct timeval tv;
1430         int32_t ticks_to_run, i, error, tick_now, interum_tick;
1431         int32_t paced_cnt = 0;
1432         int32_t did_prefetch = 0;
1433         int32_t prefetch_ninp = 0;
1434         int32_t prefetch_tp = 0;
1435         uint32_t cts;
1436         int16_t set_cpu;
1437
1438         HPTS_MTX_ASSERT(hpts);
1439         hpts->p_curtick = tcp_tv_to_hptstick(ctick);
1440         cts = tcp_tv_to_usectick(ctick);
1441         memcpy(&tv, ctick, sizeof(struct timeval));
1442         hpts->p_cur_slot = hpts_tick(hpts, 1);
1443
1444         /* Figure out if we had missed ticks */
1445 again:
1446         HPTS_MTX_ASSERT(hpts);
1447         ticks_to_run = tcp_hpts_est_run(hpts);
1448         if (!TAILQ_EMPTY(&hpts->p_input)) {
1449                 tcp_input_data(hpts, &tv);
1450         }
1451 #ifdef INVARIANTS
1452         if (TAILQ_EMPTY(&hpts->p_input) &&
1453             (hpts->p_on_inqueue_cnt != 0)) {
1454                 panic("tp:%p in_hpts input empty but cnt:%d",
1455                     hpts, hpts->p_on_inqueue_cnt);
1456         }
1457 #endif
1458         HPTS_MTX_ASSERT(hpts);
1459         /* Reset the ticks to run and time if we need too */
1460         interum_tick = tcp_gethptstick(&tv);
1461         if (interum_tick != hpts->p_curtick) {
1462                 /* Save off the new time we execute to */
1463                 *ctick = tv;
1464                 hpts->p_curtick = interum_tick;
1465                 cts = tcp_tv_to_usectick(&tv);
1466                 hpts->p_cur_slot = hpts_tick(hpts, 1);
1467                 ticks_to_run = tcp_hpts_est_run(hpts);
1468         }
1469         if (ticks_to_run == -1) {
1470                 goto no_run;
1471         }
1472         if (logging_on) {
1473                 tcp_hpts_log_it(hpts, inp, HPTSLOG_SETTORUN, ticks_to_run, 0);
1474         }
1475         if (hpts->p_on_queue_cnt == 0) {
1476                 goto no_one;
1477         }
1478         HPTS_MTX_ASSERT(hpts);
1479         for (i = 0; i < ticks_to_run; i++) {
1480                 /*
1481                  * Calculate our delay, if there are no extra ticks there
1482                  * was not any
1483                  */
1484                 hpts->p_delayed_by = (ticks_to_run - (i + 1)) * HPTS_TICKS_PER_USEC;
1485                 HPTS_MTX_ASSERT(hpts);
1486                 while ((inp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) {
1487                         /* For debugging */
1488                         if (logging_on) {
1489                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_HPTSI, ticks_to_run, i);
1490                         }
1491                         hpts->p_inp = inp;
1492                         paced_cnt++;
1493                         if (hpts->p_cur_slot != inp->inp_hptsslot) {
1494                                 panic("Hpts:%p inp:%p slot mis-aligned %u vs %u",
1495                                     hpts, inp, hpts->p_cur_slot, inp->inp_hptsslot);
1496                         }
1497                         /* Now pull it */
1498                         if (inp->inp_hpts_cpu_set == 0) {
1499                                 set_cpu = 1;
1500                         } else {
1501                                 set_cpu = 0;
1502                         }
1503                         hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[hpts->p_cur_slot], 0);
1504                         if ((ninp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) {
1505                                 /* We prefetch the next inp if possible */
1506                                 kern_prefetch(ninp, &prefetch_ninp);
1507                                 prefetch_ninp = 1;
1508                         }
1509                         if (inp->inp_hpts_request) {
1510                                 /*
1511                                  * This guy is deferred out further in time
1512                                  * then our wheel had on it. Push him back
1513                                  * on the wheel.
1514                                  */
1515                                 int32_t remaining_slots;
1516
1517                                 remaining_slots = ticks_to_run - (i + 1);
1518                                 if (inp->inp_hpts_request > remaining_slots) {
1519                                         /*
1520                                          * Keep INVARIANTS happy by clearing
1521                                          * the flag
1522                                          */
1523                                         tcp_hpts_insert_locked(hpts, inp, inp->inp_hpts_request, cts, __LINE__, NULL, 1);
1524                                         hpts->p_inp = NULL;
1525                                         continue;
1526                                 }
1527                                 inp->inp_hpts_request = 0;
1528                         }
1529                         /*
1530                          * We clear the hpts flag here after dealing with
1531                          * remaining slots. This way anyone looking with the
1532                          * TCB lock will see its on the hpts until just
1533                          * before we unlock.
1534                          */
1535                         inp->inp_in_hpts = 0;
1536                         mtx_unlock(&hpts->p_mtx);
1537                         INP_WLOCK(inp);
1538                         if (in_pcbrele_wlocked(inp)) {
1539                                 mtx_lock(&hpts->p_mtx);
1540                                 if (logging_on)
1541                                         tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 1);
1542                                 hpts->p_inp = NULL;
1543                                 continue;
1544                         }
1545                         if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
1546 out_now:
1547 #ifdef INVARIANTS
1548                                 if (mtx_owned(&hpts->p_mtx)) {
1549                                         panic("Hpts:%p owns mtx prior-to lock line:%d",
1550                                             hpts, __LINE__);
1551                                 }
1552 #endif
1553                                 INP_WUNLOCK(inp);
1554                                 mtx_lock(&hpts->p_mtx);
1555                                 if (logging_on)
1556                                         tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 3);
1557                                 hpts->p_inp = NULL;
1558                                 continue;
1559                         }
1560                         tp = intotcpcb(inp);
1561                         if ((tp == NULL) || (tp->t_inpcb == NULL)) {
1562                                 goto out_now;
1563                         }
1564                         if (set_cpu) {
1565                                 /*
1566                                  * Setup so the next time we will move to
1567                                  * the right CPU. This should be a rare
1568                                  * event. It will sometimes happens when we
1569                                  * are the client side (usually not the
1570                                  * server). Somehow tcp_output() gets called
1571                                  * before the tcp_do_segment() sets the
1572                                  * intial state. This means the r_cpu and
1573                                  * r_hpts_cpu is 0. We get on the hpts, and
1574                                  * then tcp_input() gets called setting up
1575                                  * the r_cpu to the correct value. The hpts
1576                                  * goes off and sees the mis-match. We
1577                                  * simply correct it here and the CPU will
1578                                  * switch to the new hpts nextime the tcb
1579                                  * gets added to the the hpts (not this one)
1580                                  * :-)
1581                                  */
1582                                 tcp_set_hpts(inp);
1583                         }
1584                         if (out_newts_every_tcb) {
1585                                 struct timeval sv;
1586
1587                                 if (out_ts_percision)
1588                                         microuptime(&sv);
1589                                 else
1590                                         getmicrouptime(&sv);
1591                                 cts = tcp_tv_to_usectick(&sv);
1592                         }
1593                         CURVNET_SET(tp->t_vnet);
1594                         /*
1595                          * There is a hole here, we get the refcnt on the
1596                          * inp so it will still be preserved but to make
1597                          * sure we can get the INP we need to hold the p_mtx
1598                          * above while we pull out the tp/inp,  as long as
1599                          * fini gets the lock first we are assured of having
1600                          * a sane INP we can lock and test.
1601                          */
1602 #ifdef INVARIANTS
1603                         if (mtx_owned(&hpts->p_mtx)) {
1604                                 panic("Hpts:%p owns mtx before tcp-output:%d",
1605                                     hpts, __LINE__);
1606                         }
1607 #endif
1608                         if (tp->t_fb_ptr != NULL) {
1609                                 kern_prefetch(tp->t_fb_ptr, &did_prefetch);
1610                                 did_prefetch = 1;
1611                         }
1612                         inp->inp_hpts_calls = 1;
1613                         if (tp->t_fb->tfb_tcp_output_wtime != NULL) {
1614                                 error = (*tp->t_fb->tfb_tcp_output_wtime) (tp, &tv);
1615                         } else {
1616                                 error = tp->t_fb->tfb_tcp_output(tp);
1617                         }
1618                         if (ninp && ninp->inp_ppcb) {
1619                                 /*
1620                                  * If we have a nxt inp, see if we can
1621                                  * prefetch its ppcb. Note this may seem
1622                                  * "risky" since we have no locks (other
1623                                  * than the previous inp) and there no
1624                                  * assurance that ninp was not pulled while
1625                                  * we were processing inp and freed. If this
1626                                  * occured it could mean that either:
1627                                  *
1628                                  * a) Its NULL (which is fine we won't go
1629                                  * here) <or> b) Its valid (which is cool we
1630                                  * will prefetch it) <or> c) The inp got
1631                                  * freed back to the slab which was
1632                                  * reallocated. Then the piece of memory was
1633                                  * re-used and something else (not an
1634                                  * address) is in inp_ppcb. If that occurs
1635                                  * we don't crash, but take a TLB shootdown
1636                                  * performance hit (same as if it was NULL
1637                                  * and we tried to pre-fetch it).
1638                                  *
1639                                  * Considering that the likelyhood of <c> is
1640                                  * quite rare we will take a risk on doing
1641                                  * this. If performance drops after testing
1642                                  * we can always take this out. NB: the
1643                                  * kern_prefetch on amd64 actually has
1644                                  * protection against a bad address now via
1645                                  * the DMAP_() tests. This will prevent the
1646                                  * TLB hit, and instead if <c> occurs just
1647                                  * cause us to load cache with a useless
1648                                  * address (to us).
1649                                  */
1650                                 kern_prefetch(ninp->inp_ppcb, &prefetch_tp);
1651                                 prefetch_tp = 1;
1652                         }
1653                         INP_WUNLOCK(inp);
1654                         INP_UNLOCK_ASSERT(inp);
1655                         CURVNET_RESTORE();
1656 #ifdef INVARIANTS
1657                         if (mtx_owned(&hpts->p_mtx)) {
1658                                 panic("Hpts:%p owns mtx prior-to lock line:%d",
1659                                     hpts, __LINE__);
1660                         }
1661 #endif
1662                         mtx_lock(&hpts->p_mtx);
1663                         if (logging_on)
1664                                 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 4);
1665                         hpts->p_inp = NULL;
1666                 }
1667                 HPTS_MTX_ASSERT(hpts);
1668                 hpts->p_inp = NULL;
1669                 hpts->p_cur_slot++;
1670                 if (hpts->p_cur_slot >= NUM_OF_HPTSI_SLOTS) {
1671                         hpts->p_cur_slot = 0;
1672                 }
1673         }
1674 no_one:
1675         HPTS_MTX_ASSERT(hpts);
1676         hpts->p_prevtick = hpts->p_curtick;
1677         hpts->p_delayed_by = 0;
1678         /*
1679          * Check to see if we took an excess amount of time and need to run
1680          * more ticks (if we did not hit eno-bufs).
1681          */
1682         /* Re-run any input that may be there */
1683         (void)tcp_gethptstick(&tv);
1684         if (!TAILQ_EMPTY(&hpts->p_input)) {
1685                 tcp_input_data(hpts, &tv);
1686         }
1687 #ifdef INVARIANTS
1688         if (TAILQ_EMPTY(&hpts->p_input) &&
1689             (hpts->p_on_inqueue_cnt != 0)) {
1690                 panic("tp:%p in_hpts input empty but cnt:%d",
1691                     hpts, hpts->p_on_inqueue_cnt);
1692         }
1693 #endif
1694         tick_now = tcp_gethptstick(&tv);
1695         if (SEQ_GT(tick_now, hpts->p_prevtick)) {
1696                 struct timeval res;
1697
1698                 /* Did we really spend a full tick or more in here? */
1699                 timersub(&tv, ctick, &res);
1700                 if (res.tv_sec || (res.tv_usec >= HPTS_TICKS_PER_USEC)) {
1701                         counter_u64_add(hpts_loops, 1);
1702                         if (logging_on) {
1703                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_TOLONG, (uint32_t) res.tv_usec, tick_now);
1704                         }
1705                         *ctick = res;
1706                         hpts->p_curtick = tick_now;
1707                         goto again;
1708                 }
1709         }
1710 no_run:
1711         {
1712                 uint32_t t = 0, i, fnd = 0;
1713
1714                 if (hpts->p_on_queue_cnt) {
1715
1716
1717                         /*
1718                          * Find next slot that is occupied and use that to
1719                          * be the sleep time.
1720                          */
1721                         for (i = 1, t = hpts->p_nxt_slot; i < NUM_OF_HPTSI_SLOTS; i++) {
1722                                 if (TAILQ_EMPTY(&hpts->p_hptss[t]) == 0) {
1723                                         fnd = 1;
1724                                         break;
1725                                 }
1726                                 t = (t + 1) % NUM_OF_HPTSI_SLOTS;
1727                         }
1728                         if (fnd) {
1729                                 hpts->p_hpts_sleep_time = i;
1730                         } else {
1731                                 counter_u64_add(back_tosleep, 1);
1732 #ifdef INVARIANTS
1733                                 panic("Hpts:%p cnt:%d but non found", hpts, hpts->p_on_queue_cnt);
1734 #endif
1735                                 hpts->p_on_queue_cnt = 0;
1736                                 goto non_found;
1737                         }
1738                         t++;
1739                 } else {
1740                         /* No one on the wheel sleep for all but 2 slots  */
1741 non_found:
1742                         if (hpts_sleep_max == 0)
1743                                 hpts_sleep_max = 1;
1744                         hpts->p_hpts_sleep_time = min((NUM_OF_HPTSI_SLOTS - 2), hpts_sleep_max);
1745                         t = 0;
1746                 }
1747                 if (logging_on) {
1748                         tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEPSET, t, (hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC));
1749                 }
1750         }
1751 }
1752
1753 void
1754 __tcp_set_hpts(struct inpcb *inp, int32_t line)
1755 {
1756         struct tcp_hpts_entry *hpts;
1757
1758         INP_WLOCK_ASSERT(inp);
1759         hpts = tcp_hpts_lock(inp);
1760         if ((inp->inp_in_hpts == 0) &&
1761             (inp->inp_hpts_cpu_set == 0)) {
1762                 inp->inp_hpts_cpu = hpts_cpuid(inp);
1763                 inp->inp_hpts_cpu_set = 1;
1764         }
1765         mtx_unlock(&hpts->p_mtx);
1766         hpts = tcp_input_lock(inp);
1767         if ((inp->inp_input_cpu_set == 0) &&
1768             (inp->inp_in_input == 0)) {
1769                 inp->inp_input_cpu = hpts_cpuid(inp);
1770                 inp->inp_input_cpu_set = 1;
1771         }
1772         mtx_unlock(&hpts->p_mtx);
1773 }
1774
1775 uint16_t
1776 tcp_hpts_delayedby(struct inpcb *inp){
1777         return (tcp_pace.rp_ent[inp->inp_hpts_cpu]->p_delayed_by);
1778 }
1779
1780 static void
1781 tcp_hpts_thread(void *ctx)
1782 {
1783         struct tcp_hpts_entry *hpts;
1784         struct timeval tv;
1785         sbintime_t sb;
1786
1787         hpts = (struct tcp_hpts_entry *)ctx;
1788         mtx_lock(&hpts->p_mtx);
1789         if (hpts->p_direct_wake) {
1790                 /* Signaled by input */
1791                 if (logging_on)
1792                         tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 1, 1);
1793                 callout_stop(&hpts->co);
1794         } else {
1795                 /* Timed out */
1796                 if (callout_pending(&hpts->co) ||
1797                     !callout_active(&hpts->co)) {
1798                         if (logging_on)
1799                                 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 2, 2);
1800                         mtx_unlock(&hpts->p_mtx);
1801                         return;
1802                 }
1803                 callout_deactivate(&hpts->co);
1804                 if (logging_on)
1805                         tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 3, 3);
1806         }
1807         hpts->p_hpts_active = 1;
1808         (void)tcp_gethptstick(&tv);
1809         tcp_hptsi(hpts, &tv);
1810         HPTS_MTX_ASSERT(hpts);
1811         tv.tv_sec = 0;
1812         tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC;
1813         if (tcp_min_hptsi_time && (tv.tv_usec < tcp_min_hptsi_time)) {
1814                 tv.tv_usec = tcp_min_hptsi_time;
1815                 hpts->p_on_min_sleep = 1;
1816         } else {
1817                 /* Clear the min sleep flag */
1818                 hpts->p_on_min_sleep = 0;
1819         }
1820         hpts->p_hpts_active = 0;
1821         sb = tvtosbt(tv);
1822         if (tcp_hpts_callout_skip_swi == 0) {
1823                 callout_reset_sbt_on(&hpts->co, sb, 0,
1824                     hpts_timeout_swi, hpts, hpts->p_cpu,
1825                     (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
1826         } else {
1827                 callout_reset_sbt_on(&hpts->co, sb, 0,
1828                     hpts_timeout_dir, hpts,
1829                     hpts->p_cpu,
1830                     C_PREL(tcp_hpts_precision));
1831         }
1832         hpts->p_direct_wake = 0;
1833         mtx_unlock(&hpts->p_mtx);
1834 }
1835
1836 #undef  timersub
1837
1838 static void
1839 tcp_init_hptsi(void *st)
1840 {
1841         int32_t i, j, error, bound = 0, created = 0;
1842         size_t sz, asz;
1843         struct timeval tv;
1844         sbintime_t sb;
1845         struct tcp_hpts_entry *hpts;
1846         char unit[16];
1847         uint32_t ncpus = mp_ncpus ? mp_ncpus : MAXCPU;
1848
1849         tcp_pace.rp_proc = NULL;
1850         tcp_pace.rp_num_hptss = ncpus;
1851         hpts_loops = counter_u64_alloc(M_WAITOK);
1852         back_tosleep = counter_u64_alloc(M_WAITOK);
1853
1854         sz = (tcp_pace.rp_num_hptss * sizeof(struct tcp_hpts_entry *));
1855         tcp_pace.rp_ent = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO);
1856         asz = sizeof(struct hptsh) * NUM_OF_HPTSI_SLOTS;
1857         for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
1858                 tcp_pace.rp_ent[i] = malloc(sizeof(struct tcp_hpts_entry),
1859                     M_TCPHPTS, M_WAITOK | M_ZERO);
1860                 tcp_pace.rp_ent[i]->p_hptss = malloc(asz,
1861                     M_TCPHPTS, M_WAITOK);
1862                 hpts = tcp_pace.rp_ent[i];
1863                 /*
1864                  * Init all the hpts structures that are not specifically
1865                  * zero'd by the allocations. Also lets attach them to the
1866                  * appropriate sysctl block as well.
1867                  */
1868                 mtx_init(&hpts->p_mtx, "tcp_hpts_lck",
1869                     "hpts", MTX_DEF | MTX_DUPOK);
1870                 TAILQ_INIT(&hpts->p_input);
1871                 for (j = 0; j < NUM_OF_HPTSI_SLOTS; j++) {
1872                         TAILQ_INIT(&hpts->p_hptss[j]);
1873                 }
1874                 sysctl_ctx_init(&hpts->hpts_ctx);
1875                 sprintf(unit, "%d", i);
1876                 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx,
1877                     SYSCTL_STATIC_CHILDREN(_net_inet_tcp_hpts),
1878                     OID_AUTO,
1879                     unit,
1880                     CTLFLAG_RW, 0,
1881                     "");
1882                 SYSCTL_ADD_INT(&hpts->hpts_ctx,
1883                     SYSCTL_CHILDREN(hpts->hpts_root),
1884                     OID_AUTO, "in_qcnt", CTLFLAG_RD,
1885                     &hpts->p_on_inqueue_cnt, 0,
1886                     "Count TCB's awaiting input processing");
1887                 SYSCTL_ADD_INT(&hpts->hpts_ctx,
1888                     SYSCTL_CHILDREN(hpts->hpts_root),
1889                     OID_AUTO, "out_qcnt", CTLFLAG_RD,
1890                     &hpts->p_on_queue_cnt, 0,
1891                     "Count TCB's awaiting output processing");
1892                 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1893                     SYSCTL_CHILDREN(hpts->hpts_root),
1894                     OID_AUTO, "active", CTLFLAG_RD,
1895                     &hpts->p_hpts_active, 0,
1896                     "Is the hpts active");
1897                 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1898                     SYSCTL_CHILDREN(hpts->hpts_root),
1899                     OID_AUTO, "curslot", CTLFLAG_RD,
1900                     &hpts->p_cur_slot, 0,
1901                     "What the current slot is if active");
1902                 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1903                     SYSCTL_CHILDREN(hpts->hpts_root),
1904                     OID_AUTO, "curtick", CTLFLAG_RD,
1905                     &hpts->p_curtick, 0,
1906                     "What the current tick on if active");
1907                 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1908                     SYSCTL_CHILDREN(hpts->hpts_root),
1909                     OID_AUTO, "logsize", CTLFLAG_RD,
1910                     &hpts->p_logsize, 0,
1911                     "Hpts logging buffer size");
1912                 hpts->p_hpts_sleep_time = NUM_OF_HPTSI_SLOTS - 2;
1913                 hpts->p_num = i;
1914                 hpts->p_prevtick = hpts->p_curtick = tcp_gethptstick(&tv);
1915                 hpts->p_prevtick -= 1;
1916                 hpts->p_prevtick %= NUM_OF_HPTSI_SLOTS;
1917                 hpts->p_cpu = 0xffff;
1918                 hpts->p_nxt_slot = 1;
1919                 hpts->p_logsize = tcp_hpts_logging_size;
1920                 if (hpts->p_logsize) {
1921                         sz = (sizeof(struct hpts_log) * hpts->p_logsize);
1922                         hpts->p_log = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO);
1923                 }
1924                 callout_init(&hpts->co, 1);
1925         }
1926         /*
1927          * Now lets start ithreads to handle the hptss.
1928          */
1929         CPU_FOREACH(i) {
1930                 hpts = tcp_pace.rp_ent[i];
1931                 hpts->p_cpu = i;
1932                 error = swi_add(&hpts->ie, "hpts",
1933                     tcp_hpts_thread, (void *)hpts,
1934                     SWI_NET, INTR_MPSAFE, &hpts->ie_cookie);
1935                 if (error) {
1936                         panic("Can't add hpts:%p i:%d err:%d",
1937                             hpts, i, error);
1938                 }
1939                 created++;
1940                 if (tcp_bind_threads) {
1941                         if (intr_event_bind(hpts->ie, i) == 0)
1942                                 bound++;
1943                 }
1944                 tv.tv_sec = 0;
1945                 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC;
1946                 sb = tvtosbt(tv);
1947                 if (tcp_hpts_callout_skip_swi == 0) {
1948                         callout_reset_sbt_on(&hpts->co, sb, 0,
1949                             hpts_timeout_swi, hpts, hpts->p_cpu,
1950                             (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
1951                 } else {
1952                         callout_reset_sbt_on(&hpts->co, sb, 0,
1953                             hpts_timeout_dir, hpts,
1954                             hpts->p_cpu,
1955                             C_PREL(tcp_hpts_precision));
1956                 }
1957         }
1958         printf("TCP Hpts created %d swi interrupt thread and bound %d\n",
1959             created, bound);
1960         return;
1961 }
1962
1963 SYSINIT(tcphptsi, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, tcp_init_hptsi, NULL);