sys/netinet/tcp_hpts.c

   1 /*-
   2  * Copyright (c) 2016-2018 Netflix Inc.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  16  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  23  * SUCH DAMAGE.
  24  *
  25  */
  26 #include <sys/cdefs.h>
  27 __FBSDID("$FreeBSD$");
  28
  29 #include "opt_inet.h"
  30 #include "opt_inet6.h"
  31 #include "opt_tcpdebug.h"
  32 /**
  33  * Some notes about usage.
  34  *
  35  * The tcp_hpts system is designed to provide a high precision timer
  36  * system for tcp. Its main purpose is to provide a mechanism for
  37  * pacing packets out onto the wire. It can be used in two ways
  38  * by a given TCP stack (and those two methods can be used simultaneously).
  39  *
  40  * First, and probably the main thing its used by Rack and BBR for, it can
  41  * be used to call tcp_output() of a transport stack at some time in the future.
  42  * The normal way this is done is that tcp_output() of the stack schedules
  43  * itself to be called again by calling tcp_hpts_insert(tcpcb, slot). The
  44  * slot is the time from now that the stack wants to be called but it
  45  * must be converted to tcp_hpts's notion of slot. This is done with
  46  * one of the macros HPTS_MS_TO_SLOTS or HPTS_USEC_TO_SLOTS. So a typical
  47  * call from the tcp_output() routine might look like:
  48  *
  49  * tcp_hpts_insert(tp, HPTS_USEC_TO_SLOTS(550));
  50  *
  51  * The above would schedule tcp_ouput() to be called in 550 useconds.
  52  * Note that if using this mechanism the stack will want to add near
  53  * its top a check to prevent unwanted calls (from user land or the
  54  * arrival of incoming ack's). So it would add something like:
  55  *
  56  * if (inp->inp_in_hpts)
  57  *    return;
  58  *
  59  * to prevent output processing until the time alotted has gone by.
  60  * Of course this is a bare bones example and the stack will probably
  61  * have more consideration then just the above.
  62  *
  63  * Now the tcp_hpts system will call tcp_output in one of two forms,
  64  * it will first check to see if the stack as defined a
  65  * tfb_tcp_output_wtime() function, if so that is the routine it
  66  * will call, if that function is not defined then it will call the
  67  * tfb_tcp_output() function. The only difference between these
  68  * two calls is that the former passes the time in to the function
  69  * so the function does not have to access the time (which tcp_hpts
  70  * already has). What these functions do is of course totally up
  71  * to the individual tcp stack.
  72  *
  73  * Now the second function (actually two functions I guess :D)
  74  * the tcp_hpts system provides is the  ability to either abort
  75  * a connection (later) or process  input on a connection.
  76  * Why would you want to do this? To keep processor locality.
  77  *
  78  * So in order to use the input redirection function the
  79  * stack changes its tcp_do_segment() routine to instead
  80  * of process the data call the function:
  81  *
  82  * tcp_queue_pkt_to_input()
  83  *
  84  * You will note that the arguments to this function look
  85  * a lot like tcp_do_segments's arguments. This function
  86  * will assure that the tcp_hpts system will
  87  * call the functions tfb_tcp_hpts_do_segment() from the
  88  * correct CPU. Note that multiple calls can get pushed
  89  * into the tcp_hpts system this will be indicated by
  90  * the next to last argument to tfb_tcp_hpts_do_segment()
  91  * (nxt_pkt). If nxt_pkt is a 1 then another packet is
  92  * coming. If nxt_pkt is a 0 then this is the last call
  93  * that the tcp_hpts system has available for the tcp stack.
  94  *
  95  * The other point of the input system is to be able to safely
  96  * drop a tcp connection without worrying about the recursive
  97  * locking that may be occuring on the INP_WLOCK. So if
  98  * a stack wants to drop a connection it calls:
  99  *
 100  *     tcp_set_inp_to_drop(tp, ETIMEDOUT)
 101  *
 102  * To schedule the tcp_hpts system to call
 103  *
 104  *    tcp_drop(tp, drop_reason)
 105  *
 106  * at a future point. This is quite handy to prevent locking
 107  * issues when dropping connections.
 108  *
 109  */
 110
 111 #include <sys/param.h>
 112 #include <sys/bus.h>
 113 #include <sys/interrupt.h>
 114 #include <sys/module.h>
 115 #include <sys/kernel.h>
 116 #include <sys/hhook.h>
 117 #include <sys/malloc.h>
 118 #include <sys/mbuf.h>
 119 #include <sys/proc.h>           /* for proc0 declaration */
 120 #include <sys/socket.h>
 121 #include <sys/socketvar.h>
 122 #include <sys/sysctl.h>
 123 #include <sys/systm.h>
 124 #include <sys/refcount.h>
 125 #include <sys/sched.h>
 126 #include <sys/queue.h>
 127 #include <sys/smp.h>
 128 #include <sys/counter.h>
 129 #include <sys/time.h>
 130 #include <sys/kthread.h>
 131 #include <sys/kern_prefetch.h>
 132
 133 #include <vm/uma.h>
 134
 135 #include <net/route.h>
 136 #include <net/vnet.h>
 137
 138 #define TCPSTATES               /* for logging */
 139
 140 #include <netinet/in.h>
 141 #include <netinet/in_kdtrace.h>
 142 #include <netinet/in_pcb.h>
 143 #include <netinet/ip.h>
 144 #include <netinet/ip_icmp.h>    /* required for icmp_var.h */
 145 #include <netinet/icmp_var.h>   /* for ICMP_BANDLIM */
 146 #include <netinet/ip_var.h>
 147 #include <netinet/ip6.h>
 148 #include <netinet6/in6_pcb.h>
 149 #include <netinet6/ip6_var.h>
 150 #include <netinet/tcp.h>
 151 #include <netinet/tcp_fsm.h>
 152 #include <netinet/tcp_seq.h>
 153 #include <netinet/tcp_timer.h>
 154 #include <netinet/tcp_var.h>
 155 #include <netinet/tcpip.h>
 156 #include <netinet/cc/cc.h>
 157 #include <netinet/tcp_hpts.h>
 158
 159 #ifdef tcpdebug
 160 #include <netinet/tcp_debug.h>
 161 #endif                          /* tcpdebug */
 162 #ifdef tcp_offload
 163 #include <netinet/tcp_offload.h>
 164 #endif
 165
 166 #include "opt_rss.h"
 167
 168 MALLOC_DEFINE(M_TCPHPTS, "tcp_hpts", "TCP hpts");
 169 #ifdef RSS
 170 static int tcp_bind_threads = 1;
 171 #else
 172 static int tcp_bind_threads = 0;
 173 #endif
 174 TUNABLE_INT("net.inet.tcp.bind_hptss", &tcp_bind_threads);
 175
 176 static uint32_t tcp_hpts_logging_size = DEFAULT_HPTS_LOG;
 177
 178 TUNABLE_INT("net.inet.tcp.hpts_logging_sz", &tcp_hpts_logging_size);
 179
 180 static struct tcp_hptsi tcp_pace;
 181
 182 static void tcp_wakehpts(struct tcp_hpts_entry *p);
 183 static void tcp_wakeinput(struct tcp_hpts_entry *p);
 184 static void tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv);
 185 static void tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick);
 186 static void tcp_hpts_thread(void *ctx);
 187 static void tcp_init_hptsi(void *st);
 188
 189 int32_t tcp_min_hptsi_time = DEFAULT_MIN_SLEEP;
 190 static int32_t tcp_hpts_callout_skip_swi = 0;
 191
 192 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hpts, CTLFLAG_RW, 0, "TCP Hpts controls");
 193
 194 #define timersub(tvp, uvp, vvp)                                         \
 195         do {                                                            \
 196                 (vvp)->tv_sec = (tvp)->tv_sec - (uvp)->tv_sec;          \
 197                 (vvp)->tv_usec = (tvp)->tv_usec - (uvp)->tv_usec;       \
 198                 if ((vvp)->tv_usec < 0) {                               \
 199                         (vvp)->tv_sec--;                                \
 200                         (vvp)->tv_usec += 1000000;                      \
 201                 }                                                       \
 202         } while (0)
 203
 204 static int32_t logging_on = 0;
 205 static int32_t hpts_sleep_max = (NUM_OF_HPTSI_SLOTS - 2);
 206 static int32_t tcp_hpts_precision = 120;
 207
 208 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, precision, CTLFLAG_RW,
 209     &tcp_hpts_precision, 120,
 210     "Value for PRE() precision of callout");
 211
 212 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, logging, CTLFLAG_RW,
 213     &logging_on, 0,
 214     "Turn on logging if compiled in");
 215
 216 counter_u64_t hpts_loops;
 217
 218 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, loops, CTLFLAG_RD,
 219     &hpts_loops, "Number of times hpts had to loop to catch up");
 220
 221 counter_u64_t back_tosleep;
 222
 223 SYSCTL_COUNTER_U64(_net_inet_tcp_hpts, OID_AUTO, no_tcbsfound, CTLFLAG_RD,
 224     &back_tosleep, "Number of times hpts found no tcbs");
 225
 226 static int32_t in_newts_every_tcb = 0;
 227
 228 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tsperpcb, CTLFLAG_RW,
 229     &in_newts_every_tcb, 0,
 230     "Do we have a new cts every tcb we process for input");
 231 static int32_t in_ts_percision = 0;
 232
 233 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, in_tspercision, CTLFLAG_RW,
 234     &in_ts_percision, 0,
 235     "Do we use percise timestamp for clients on input");
 236 static int32_t out_newts_every_tcb = 0;
 237
 238 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tsperpcb, CTLFLAG_RW,
 239     &out_newts_every_tcb, 0,
 240     "Do we have a new cts every tcb we process for output");
 241 static int32_t out_ts_percision = 0;
 242
 243 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, out_tspercision, CTLFLAG_RW,
 244     &out_ts_percision, 0,
 245     "Do we use a percise timestamp for every output cts");
 246
 247 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, maxsleep, CTLFLAG_RW,
 248     &hpts_sleep_max, 0,
 249     "The maximum time the hpts will sleep <1 - 254>");
 250
 251 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, minsleep, CTLFLAG_RW,
 252     &tcp_min_hptsi_time, 0,
 253     "The minimum time the hpts must sleep before processing more slots");
 254
 255 SYSCTL_INT(_net_inet_tcp_hpts, OID_AUTO, skip_swi, CTLFLAG_RW,
 256     &tcp_hpts_callout_skip_swi, 0,
 257     "Do we have the callout call directly to the hpts?");
 258
 259 static void
 260 __tcp_hpts_log_it(struct tcp_hpts_entry *hpts, struct inpcb *inp, int event, uint32_t slot,
 261     uint32_t ticknow, int32_t line)
 262 {
 263         struct hpts_log *pl;
 264
 265         HPTS_MTX_ASSERT(hpts);
 266         if (hpts->p_log == NULL)
 267                 return;
 268         pl = &hpts->p_log[hpts->p_log_at];
 269         hpts->p_log_at++;
 270         if (hpts->p_log_at >= hpts->p_logsize) {
 271                 hpts->p_log_at = 0;
 272                 hpts->p_log_wrapped = 1;
 273         }
 274         pl->inp = inp;
 275         if (inp) {
 276                 pl->t_paceslot = inp->inp_hptsslot;
 277                 pl->t_hptsreq = inp->inp_hpts_request;
 278                 pl->p_onhpts = inp->inp_in_hpts;
 279                 pl->p_oninput = inp->inp_in_input;
 280         } else {
 281                 pl->t_paceslot = 0;
 282                 pl->t_hptsreq = 0;
 283                 pl->p_onhpts = 0;
 284                 pl->p_oninput = 0;
 285         }
 286         pl->is_notempty = 1;
 287         pl->event = event;
 288         pl->line = line;
 289         pl->cts = tcp_get_usecs(NULL);
 290         pl->p_curtick = hpts->p_curtick;
 291         pl->p_prevtick = hpts->p_prevtick;
 292         pl->p_on_queue_cnt = hpts->p_on_queue_cnt;
 293         pl->ticknow = ticknow;
 294         pl->slot_req = slot;
 295         pl->p_nxt_slot = hpts->p_nxt_slot;
 296         pl->p_cur_slot = hpts->p_cur_slot;
 297         pl->p_hpts_sleep_time = hpts->p_hpts_sleep_time;
 298         pl->p_flags = (hpts->p_cpu & 0x7f);
 299         pl->p_flags <<= 7;
 300         pl->p_flags |= (hpts->p_num & 0x7f);
 301         pl->p_flags <<= 2;
 302         if (hpts->p_hpts_active) {
 303                 pl->p_flags |= HPTS_HPTS_ACTIVE;
 304         }
 305 }
 306
 307 #define tcp_hpts_log_it(a, b, c, d, e) __tcp_hpts_log_it(a, b, c, d, e, __LINE__)
 308
 309 static void
 310 hpts_timeout_swi(void *arg)
 311 {
 312         struct tcp_hpts_entry *hpts;
 313
 314         hpts = (struct tcp_hpts_entry *)arg;
 315         swi_sched(hpts->ie_cookie, 0);
 316 }
 317
 318 static void
 319 hpts_timeout_dir(void *arg)
 320 {
 321         tcp_hpts_thread(arg);
 322 }
 323
 324 static inline void
 325 hpts_sane_pace_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int clear)
 326 {
 327 #ifdef INVARIANTS
 328         if (mtx_owned(&hpts->p_mtx) == 0) {
 329                 /* We don't own the mutex? */
 330                 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
 331         }
 332         if (hpts->p_cpu != inp->inp_hpts_cpu) {
 333                 /* It is not the right cpu/mutex? */
 334                 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
 335         }
 336         if (inp->inp_in_hpts == 0) {
 337                 /* We are not on the hpts? */
 338                 panic("%s: hpts:%p inp:%p not on the hpts?", __FUNCTION__, hpts, inp);
 339         }
 340         if (TAILQ_EMPTY(head) &&
 341             (hpts->p_on_queue_cnt != 0)) {
 342                 /* We should not be empty with a queue count */
 343                 panic("%s hpts:%p hpts bucket empty but cnt:%d",
 344                     __FUNCTION__, hpts, hpts->p_on_queue_cnt);
 345         }
 346 #endif
 347         TAILQ_REMOVE(head, inp, inp_hpts);
 348         hpts->p_on_queue_cnt--;
 349         if (hpts->p_on_queue_cnt < 0) {
 350                 /* Count should not go negative .. */
 351 #ifdef INVARIANTS
 352                 panic("Hpts goes negative inp:%p hpts:%p",
 353                     inp, hpts);
 354 #endif
 355                 hpts->p_on_queue_cnt = 0;
 356         }
 357         if (clear) {
 358                 inp->inp_hpts_request = 0;
 359                 inp->inp_in_hpts = 0;
 360         }
 361 }
 362
 363 static inline void
 364 hpts_sane_pace_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, struct hptsh *head, int line, int noref)
 365 {
 366 #ifdef INVARIANTS
 367         if (mtx_owned(&hpts->p_mtx) == 0) {
 368                 /* We don't own the mutex? */
 369                 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
 370         }
 371         if (hpts->p_cpu != inp->inp_hpts_cpu) {
 372                 /* It is not the right cpu/mutex? */
 373                 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
 374         }
 375         if ((noref == 0) && (inp->inp_in_hpts == 1)) {
 376                 /* We are already on the hpts? */
 377                 panic("%s: hpts:%p inp:%p already on the hpts?", __FUNCTION__, hpts, inp);
 378         }
 379 #endif
 380         TAILQ_INSERT_TAIL(head, inp, inp_hpts);
 381         inp->inp_in_hpts = 1;
 382         hpts->p_on_queue_cnt++;
 383         if (noref == 0) {
 384                 in_pcbref(inp);
 385         }
 386 }
 387
 388 static inline void
 389 hpts_sane_input_remove(struct tcp_hpts_entry *hpts, struct inpcb *inp, int clear)
 390 {
 391 #ifdef INVARIANTS
 392         if (mtx_owned(&hpts->p_mtx) == 0) {
 393                 /* We don't own the mutex? */
 394                 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
 395         }
 396         if (hpts->p_cpu != inp->inp_input_cpu) {
 397                 /* It is not the right cpu/mutex? */
 398                 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
 399         }
 400         if (inp->inp_in_input == 0) {
 401                 /* We are not on the input hpts? */
 402                 panic("%s: hpts:%p inp:%p not on the input hpts?", __FUNCTION__, hpts, inp);
 403         }
 404 #endif
 405         TAILQ_REMOVE(&hpts->p_input, inp, inp_input);
 406         hpts->p_on_inqueue_cnt--;
 407         if (hpts->p_on_inqueue_cnt < 0) {
 408 #ifdef INVARIANTS
 409                 panic("Hpts in goes negative inp:%p hpts:%p",
 410                     inp, hpts);
 411 #endif
 412                 hpts->p_on_inqueue_cnt = 0;
 413         }
 414 #ifdef INVARIANTS
 415         if (TAILQ_EMPTY(&hpts->p_input) &&
 416             (hpts->p_on_inqueue_cnt != 0)) {
 417                 /* We should not be empty with a queue count */
 418                 panic("%s hpts:%p in_hpts input empty but cnt:%d",
 419                     __FUNCTION__, hpts, hpts->p_on_inqueue_cnt);
 420         }
 421 #endif
 422         if (clear)
 423                 inp->inp_in_input = 0;
 424 }
 425
 426 static inline void
 427 hpts_sane_input_insert(struct tcp_hpts_entry *hpts, struct inpcb *inp, int line)
 428 {
 429 #ifdef INVARIANTS
 430         if (mtx_owned(&hpts->p_mtx) == 0) {
 431                 /* We don't own the mutex? */
 432                 panic("%s: hpts:%p inp:%p no hpts mutex", __FUNCTION__, hpts, inp);
 433         }
 434         if (hpts->p_cpu != inp->inp_input_cpu) {
 435                 /* It is not the right cpu/mutex? */
 436                 panic("%s: hpts:%p inp:%p incorrect CPU", __FUNCTION__, hpts, inp);
 437         }
 438         if (inp->inp_in_input == 1) {
 439                 /* We are already on the input hpts? */
 440                 panic("%s: hpts:%p inp:%p already on the input hpts?", __FUNCTION__, hpts, inp);
 441         }
 442 #endif
 443         TAILQ_INSERT_TAIL(&hpts->p_input, inp, inp_input);
 444         inp->inp_in_input = 1;
 445         hpts->p_on_inqueue_cnt++;
 446         in_pcbref(inp);
 447 }
 448
 449 static int
 450 sysctl_tcp_hpts_log(SYSCTL_HANDLER_ARGS)
 451 {
 452         struct tcp_hpts_entry *hpts;
 453         size_t sz;
 454         int32_t logging_was, i;
 455         int32_t error = 0;
 456
 457         /*
 458          * HACK: Turn off logging so no locks are required this really needs
 459          * a memory barrier :)
 460          */
 461         logging_was = logging_on;
 462         logging_on = 0;
 463         if (!req->oldptr) {
 464                 /* How much? */
 465                 sz = 0;
 466                 for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
 467                         hpts = tcp_pace.rp_ent[i];
 468                         if (hpts->p_log == NULL)
 469                                 continue;
 470                         sz += (sizeof(struct hpts_log) * hpts->p_logsize);
 471                 }
 472                 error = SYSCTL_OUT(req, 0, sz);
 473         } else {
 474                 for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
 475                         hpts = tcp_pace.rp_ent[i];
 476                         if (hpts->p_log == NULL)
 477                                 continue;
 478                         if (hpts->p_log_wrapped)
 479                                 sz = (sizeof(struct hpts_log) * hpts->p_logsize);
 480                         else
 481                                 sz = (sizeof(struct hpts_log) * hpts->p_log_at);
 482                         error = SYSCTL_OUT(req, hpts->p_log, sz);
 483                 }
 484         }
 485         logging_on = logging_was;
 486         return error;
 487 }
 488
 489 SYSCTL_PROC(_net_inet_tcp_hpts, OID_AUTO, log, CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
 490     0, 0, sysctl_tcp_hpts_log, "A", "tcp hptsi log");
 491
 492
 493 static void
 494 tcp_wakehpts(struct tcp_hpts_entry *hpts)
 495 {
 496         HPTS_MTX_ASSERT(hpts);
 497         swi_sched(hpts->ie_cookie, 0);
 498         if (hpts->p_hpts_active == 2) {
 499                 /* Rare sleeping on a ENOBUF */
 500                 wakeup_one(hpts);
 501         }
 502 }
 503
 504 static void
 505 tcp_wakeinput(struct tcp_hpts_entry *hpts)
 506 {
 507         HPTS_MTX_ASSERT(hpts);
 508         swi_sched(hpts->ie_cookie, 0);
 509         if (hpts->p_hpts_active == 2) {
 510                 /* Rare sleeping on a ENOBUF */
 511                 wakeup_one(hpts);
 512         }
 513 }
 514
 515 struct tcp_hpts_entry *
 516 tcp_cur_hpts(struct inpcb *inp)
 517 {
 518         int32_t hpts_num;
 519         struct tcp_hpts_entry *hpts;
 520
 521         hpts_num = inp->inp_hpts_cpu;
 522         hpts = tcp_pace.rp_ent[hpts_num];
 523         return (hpts);
 524 }
 525
 526 struct tcp_hpts_entry *
 527 tcp_hpts_lock(struct inpcb *inp)
 528 {
 529         struct tcp_hpts_entry *hpts;
 530         int32_t hpts_num;
 531
 532 again:
 533         hpts_num = inp->inp_hpts_cpu;
 534         hpts = tcp_pace.rp_ent[hpts_num];
 535 #ifdef INVARIANTS
 536         if (mtx_owned(&hpts->p_mtx)) {
 537                 panic("Hpts:%p owns mtx prior-to lock line:%d",
 538                     hpts, __LINE__);
 539         }
 540 #endif
 541         mtx_lock(&hpts->p_mtx);
 542         if (hpts_num != inp->inp_hpts_cpu) {
 543                 mtx_unlock(&hpts->p_mtx);
 544                 goto again;
 545         }
 546         return (hpts);
 547 }
 548
 549 struct tcp_hpts_entry *
 550 tcp_input_lock(struct inpcb *inp)
 551 {
 552         struct tcp_hpts_entry *hpts;
 553         int32_t hpts_num;
 554
 555 again:
 556         hpts_num = inp->inp_input_cpu;
 557         hpts = tcp_pace.rp_ent[hpts_num];
 558 #ifdef INVARIANTS
 559         if (mtx_owned(&hpts->p_mtx)) {
 560                 panic("Hpts:%p owns mtx prior-to lock line:%d",
 561                     hpts, __LINE__);
 562         }
 563 #endif
 564         mtx_lock(&hpts->p_mtx);
 565         if (hpts_num != inp->inp_input_cpu) {
 566                 mtx_unlock(&hpts->p_mtx);
 567                 goto again;
 568         }
 569         return (hpts);
 570 }
 571
 572 static void
 573 tcp_remove_hpts_ref(struct inpcb *inp, struct tcp_hpts_entry *hpts, int line)
 574 {
 575         int32_t add_freed;
 576
 577         if (inp->inp_flags2 & INP_FREED) {
 578                 /*
 579                  * Need to play a special trick so that in_pcbrele_wlocked
 580                  * does not return 1 when it really should have returned 0.
 581                  */
 582                 add_freed = 1;
 583                 inp->inp_flags2 &= ~INP_FREED;
 584         } else {
 585                 add_freed = 0;
 586         }
 587 #ifndef INP_REF_DEBUG
 588         if (in_pcbrele_wlocked(inp)) {
 589                 /*
 590                  * This should not happen. We have the inpcb referred to by
 591                  * the main socket (why we are called) and the hpts. It
 592                  * should always return 0.
 593                  */
 594                 panic("inpcb:%p release ret 1",
 595                     inp);
 596         }
 597 #else
 598         if (__in_pcbrele_wlocked(inp, line)) {
 599                 /*
 600                  * This should not happen. We have the inpcb referred to by
 601                  * the main socket (why we are called) and the hpts. It
 602                  * should always return 0.
 603                  */
 604                 panic("inpcb:%p release ret 1",
 605                     inp);
 606         }
 607 #endif
 608         if (add_freed) {
 609                 inp->inp_flags2 |= INP_FREED;
 610         }
 611 }
 612
 613 static void
 614 tcp_hpts_remove_locked_output(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line)
 615 {
 616         if (inp->inp_in_hpts) {
 617                 hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], 1);
 618                 tcp_remove_hpts_ref(inp, hpts, line);
 619         }
 620 }
 621
 622 static void
 623 tcp_hpts_remove_locked_input(struct tcp_hpts_entry *hpts, struct inpcb *inp, int32_t flags, int32_t line)
 624 {
 625         HPTS_MTX_ASSERT(hpts);
 626         if (inp->inp_in_input) {
 627                 hpts_sane_input_remove(hpts, inp, 1);
 628                 tcp_remove_hpts_ref(inp, hpts, line);
 629         }
 630 }
 631
 632 /*
 633  * Called normally with the INP_LOCKED but it
 634  * does not matter, the hpts lock is the key
 635  * but the lock order allows us to hold the
 636  * INP lock and then get the hpts lock.
 637  *
 638  * Valid values in the flags are
 639  * HPTS_REMOVE_OUTPUT - remove from the output of the hpts.
 640  * HPTS_REMOVE_INPUT - remove from the input of the hpts.
 641  * Note that you can or both values together and get two
 642  * actions.
 643  */
 644 void
 645 __tcp_hpts_remove(struct inpcb *inp, int32_t flags, int32_t line)
 646 {
 647         struct tcp_hpts_entry *hpts;
 648
 649         INP_WLOCK_ASSERT(inp);
 650         if (flags & HPTS_REMOVE_OUTPUT) {
 651                 hpts = tcp_hpts_lock(inp);
 652                 tcp_hpts_remove_locked_output(hpts, inp, flags, line);
 653                 mtx_unlock(&hpts->p_mtx);
 654         }
 655         if (flags & HPTS_REMOVE_INPUT) {
 656                 hpts = tcp_input_lock(inp);
 657                 tcp_hpts_remove_locked_input(hpts, inp, flags, line);
 658                 mtx_unlock(&hpts->p_mtx);
 659         }
 660 }
 661
 662 static inline int
 663 hpts_tick(struct tcp_hpts_entry *hpts, int32_t plus)
 664 {
 665         return ((hpts->p_prevtick + plus) % NUM_OF_HPTSI_SLOTS);
 666 }
 667
 668 static int
 669 tcp_queue_to_hpts_immediate_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line, int32_t noref)
 670 {
 671         int32_t need_wake = 0;
 672         uint32_t ticknow = 0;
 673
 674         HPTS_MTX_ASSERT(hpts);
 675         if (inp->inp_in_hpts == 0) {
 676                 /* Ok we need to set it on the hpts in the current slot */
 677                 if (hpts->p_hpts_active == 0) {
 678                         /* A sleeping hpts we want in next slot to run */
 679                         if (logging_on) {
 680                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, 0,
 681                                     hpts_tick(hpts, 1));
 682                         }
 683                         inp->inp_hptsslot = hpts_tick(hpts, 1);
 684                         inp->inp_hpts_request = 0;
 685                         if (logging_on) {
 686                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEP_BEFORE, 1, ticknow);
 687                         }
 688                         need_wake = 1;
 689                 } else if ((void *)inp == hpts->p_inp) {
 690                         /*
 691                          * We can't allow you to go into the same slot we
 692                          * are in. We must put you out.
 693                          */
 694                         inp->inp_hptsslot = hpts->p_nxt_slot;
 695                 } else
 696                         inp->inp_hptsslot = hpts->p_cur_slot;
 697                 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref);
 698                 inp->inp_hpts_request = 0;
 699                 if (logging_on) {
 700                         tcp_hpts_log_it(hpts, inp, HPTSLOG_IMMEDIATE, 0, 0);
 701                 }
 702                 if (need_wake) {
 703                         /*
 704                          * Activate the hpts if it is sleeping and its
 705                          * timeout is not 1.
 706                          */
 707                         if (logging_on) {
 708                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_HPTS, 0, ticknow);
 709                         }
 710                         hpts->p_direct_wake = 1;
 711                         tcp_wakehpts(hpts);
 712                 }
 713         }
 714         return (need_wake);
 715 }
 716
 717 int
 718 __tcp_queue_to_hpts_immediate(struct inpcb *inp, int32_t line)
 719 {
 720         int32_t ret;
 721         struct tcp_hpts_entry *hpts;
 722
 723         INP_WLOCK_ASSERT(inp);
 724         hpts = tcp_hpts_lock(inp);
 725         ret = tcp_queue_to_hpts_immediate_locked(inp, hpts, line, 0);
 726         mtx_unlock(&hpts->p_mtx);
 727         return (ret);
 728 }
 729
 730 static void
 731 tcp_hpts_insert_locked(struct tcp_hpts_entry *hpts, struct inpcb *inp, uint32_t slot, uint32_t cts, int32_t line,
 732     struct hpts_diag *diag, int32_t noref)
 733 {
 734         int32_t need_new_to = 0;
 735         int32_t need_wakeup = 0;
 736         uint32_t largest_slot;
 737         uint32_t ticknow = 0;
 738         uint32_t slot_calc;
 739
 740         HPTS_MTX_ASSERT(hpts);
 741         if (diag) {
 742                 memset(diag, 0, sizeof(struct hpts_diag));
 743                 diag->p_hpts_active = hpts->p_hpts_active;
 744                 diag->p_nxt_slot = hpts->p_nxt_slot;
 745                 diag->p_cur_slot = hpts->p_cur_slot;
 746                 diag->slot_req = slot;
 747         }
 748         if ((inp->inp_in_hpts == 0) || noref) {
 749                 inp->inp_hpts_request = slot;
 750                 if (slot == 0) {
 751                         /* Immediate */
 752                         tcp_queue_to_hpts_immediate_locked(inp, hpts, line, noref);
 753                         return;
 754                 }
 755                 if (hpts->p_hpts_active) {
 756                         /*
 757                          * Its slot - 1 since nxt_slot is the next tick that
 758                          * will go off since the hpts is awake
 759                          */
 760                         if (logging_on) {
 761                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_NORMAL, slot, 0);
 762                         }
 763                         /*
 764                          * We want to make sure that we don't place a inp in
 765                          * the range of p_cur_slot <-> p_nxt_slot. If we
 766                          * take from p_nxt_slot to the end, plus p_cur_slot
 767                          * and then take away 2, we will know how many is
 768                          * the max slots we can use.
 769                          */
 770                         if (hpts->p_nxt_slot > hpts->p_cur_slot) {
 771                                 /*
 772                                  * Non-wrap case nxt_slot <-> cur_slot we
 773                                  * don't want to land in. So the diff gives
 774                                  * us what is taken away from the number of
 775                                  * slots.
 776                                  */
 777                                 largest_slot = NUM_OF_HPTSI_SLOTS - (hpts->p_nxt_slot - hpts->p_cur_slot);
 778                         } else if (hpts->p_nxt_slot == hpts->p_cur_slot) {
 779                                 largest_slot = NUM_OF_HPTSI_SLOTS - 2;
 780                         } else {
 781                                 /*
 782                                  * Wrap case so the diff gives us the number
 783                                  * of slots that we can land in.
 784                                  */
 785                                 largest_slot = hpts->p_cur_slot - hpts->p_nxt_slot;
 786                         }
 787                         /*
 788                          * We take away two so we never have a problem (20
 789                          * usec's) out of 1024000 usecs
 790                          */
 791                         largest_slot -= 2;
 792                         if (inp->inp_hpts_request > largest_slot) {
 793                                 /*
 794                                  * Restrict max jump of slots and remember
 795                                  * leftover
 796                                  */
 797                                 slot = largest_slot;
 798                                 inp->inp_hpts_request -= largest_slot;
 799                         } else {
 800                                 /* This one will run when we hit it */
 801                                 inp->inp_hpts_request = 0;
 802                         }
 803                         if (hpts->p_nxt_slot == hpts->p_cur_slot)
 804                                 slot_calc = (hpts->p_nxt_slot + slot) % NUM_OF_HPTSI_SLOTS;
 805                         else
 806                                 slot_calc = (hpts->p_nxt_slot + slot - 1) % NUM_OF_HPTSI_SLOTS;
 807                         if (slot_calc == hpts->p_cur_slot) {
 808 #ifdef INVARIANTS
 809                                 /* TSNH */
 810                                 panic("Hpts:%p impossible slot calculation slot_calc:%u slot:%u largest:%u\n",
 811                                     hpts, slot_calc, slot, largest_slot);
 812 #endif
 813                                 if (slot_calc)
 814                                         slot_calc--;
 815                                 else
 816                                         slot_calc = NUM_OF_HPTSI_SLOTS - 1;
 817                         }
 818                         inp->inp_hptsslot = slot_calc;
 819                         if (diag) {
 820                                 diag->inp_hptsslot = inp->inp_hptsslot;
 821                         }
 822                 } else {
 823                         /*
 824                          * The hpts is sleeping, we need to figure out where
 825                          * it will wake up at and if we need to reschedule
 826                          * its time-out.
 827                          */
 828                         uint32_t have_slept, yet_to_sleep;
 829                         uint32_t slot_now;
 830                         struct timeval tv;
 831
 832                         ticknow = tcp_gethptstick(&tv);
 833                         slot_now = ticknow % NUM_OF_HPTSI_SLOTS;
 834                         /*
 835                          * The user wants to be inserted at (slot_now +
 836                          * slot) % NUM_OF_HPTSI_SLOTS, so lets set that up.
 837                          */
 838                         largest_slot = NUM_OF_HPTSI_SLOTS - 2;
 839                         if (inp->inp_hpts_request > largest_slot) {
 840                                 /* Adjust the residual in inp_hpts_request */
 841                                 slot = largest_slot;
 842                                 inp->inp_hpts_request -= largest_slot;
 843                         } else {
 844                                 /* No residual it all fits */
 845                                 inp->inp_hpts_request = 0;
 846                         }
 847                         inp->inp_hptsslot = (slot_now + slot) % NUM_OF_HPTSI_SLOTS;
 848                         if (diag) {
 849                                 diag->slot_now = slot_now;
 850                                 diag->inp_hptsslot = inp->inp_hptsslot;
 851                                 diag->p_on_min_sleep = hpts->p_on_min_sleep;
 852                         }
 853                         if (logging_on) {
 854                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERT_SLEEPER, slot, ticknow);
 855                         }
 856                         /* Now do we need to restart the hpts's timer? */
 857                         if (TSTMP_GT(ticknow, hpts->p_curtick))
 858                                 have_slept = ticknow - hpts->p_curtick;
 859                         else
 860                                 have_slept = 0;
 861                         if (have_slept < hpts->p_hpts_sleep_time) {
 862                                 /* This should be what happens */
 863                                 yet_to_sleep = hpts->p_hpts_sleep_time - have_slept;
 864                         } else {
 865                                 /* We are over-due */
 866                                 yet_to_sleep = 0;
 867                                 need_wakeup = 1;
 868                         }
 869                         if (diag) {
 870                                 diag->have_slept = have_slept;
 871                                 diag->yet_to_sleep = yet_to_sleep;
 872                                 diag->hpts_sleep_time = hpts->p_hpts_sleep_time;
 873                         }
 874                         if ((hpts->p_on_min_sleep == 0) && (yet_to_sleep > slot)) {
 875                                 /*
 876                                  * We need to reschedule the hptss time-out.
 877                                  */
 878                                 hpts->p_hpts_sleep_time = slot;
 879                                 need_new_to = slot * HPTS_TICKS_PER_USEC;
 880                         }
 881                 }
 882                 hpts_sane_pace_insert(hpts, inp, &hpts->p_hptss[inp->inp_hptsslot], line, noref);
 883                 if (logging_on) {
 884                         tcp_hpts_log_it(hpts, inp, HPTSLOG_INSERTED, slot, ticknow);
 885                 }
 886                 /*
 887                  * Now how far is the hpts sleeping to? if active is 1, its
 888                  * up and ticking we do nothing, otherwise we may need to
 889                  * reschedule its callout if need_new_to is set from above.
 890                  */
 891                 if (need_wakeup) {
 892                         if (logging_on) {
 893                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_RESCHEDULE, 1, 0);
 894                         }
 895                         hpts->p_direct_wake = 1;
 896                         tcp_wakehpts(hpts);
 897                         if (diag) {
 898                                 diag->need_new_to = 0;
 899                                 diag->co_ret = 0xffff0000;
 900                         }
 901                 } else if (need_new_to) {
 902                         int32_t co_ret;
 903                         struct timeval tv;
 904                         sbintime_t sb;
 905
 906                         tv.tv_sec = 0;
 907                         tv.tv_usec = 0;
 908                         while (need_new_to > HPTS_USEC_IN_SEC) {
 909                                 tv.tv_sec++;
 910                                 need_new_to -= HPTS_USEC_IN_SEC;
 911                         }
 912                         tv.tv_usec = need_new_to;
 913                         sb = tvtosbt(tv);
 914                         if (tcp_hpts_callout_skip_swi == 0) {
 915                                 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0,
 916                                     hpts_timeout_swi, hpts, hpts->p_cpu,
 917                                     (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
 918                         } else {
 919                                 co_ret = callout_reset_sbt_on(&hpts->co, sb, 0,
 920                                     hpts_timeout_dir, hpts,
 921                                     hpts->p_cpu,
 922                                     C_PREL(tcp_hpts_precision));
 923                         }
 924                         if (diag) {
 925                                 diag->need_new_to = need_new_to;
 926                                 diag->co_ret = co_ret;
 927                         }
 928                 }
 929         } else {
 930 #ifdef INVARIANTS
 931                 panic("Hpts:%p tp:%p already on hpts and add?", hpts, inp);
 932 #endif
 933         }
 934 }
 935
 936 uint32_t
 937 tcp_hpts_insert_diag(struct inpcb *inp, uint32_t slot, int32_t line, struct hpts_diag *diag){
 938         struct tcp_hpts_entry *hpts;
 939         uint32_t slot_on, cts;
 940         struct timeval tv;
 941
 942         /*
 943          * We now return the next-slot the hpts will be on, beyond its
 944          * current run (if up) or where it was when it stopped if it is
 945          * sleeping.
 946          */
 947         INP_WLOCK_ASSERT(inp);
 948         hpts = tcp_hpts_lock(inp);
 949         if (in_ts_percision)
 950                 microuptime(&tv);
 951         else
 952                 getmicrouptime(&tv);
 953         cts = tcp_tv_to_usectick(&tv);
 954         tcp_hpts_insert_locked(hpts, inp, slot, cts, line, diag, 0);
 955         slot_on = hpts->p_nxt_slot;
 956         mtx_unlock(&hpts->p_mtx);
 957         return (slot_on);
 958 }
 959
 960 uint32_t
 961 __tcp_hpts_insert(struct inpcb *inp, uint32_t slot, int32_t line){
 962         return (tcp_hpts_insert_diag(inp, slot, line, NULL));
 963 }
 964
 965 int
 966 __tcp_queue_to_input_locked(struct inpcb *inp, struct tcp_hpts_entry *hpts, int32_t line)
 967 {
 968         int32_t retval = 0;
 969
 970         HPTS_MTX_ASSERT(hpts);
 971         if (inp->inp_in_input == 0) {
 972                 /* Ok we need to set it on the hpts in the current slot */
 973                 hpts_sane_input_insert(hpts, inp, line);
 974                 retval = 1;
 975                 if (hpts->p_hpts_active == 0) {
 976                         /*
 977                          * Activate the hpts if it is sleeping.
 978                          */
 979                         if (logging_on) {
 980                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_WAKEUP_INPUT, 0, 0);
 981                         }
 982                         retval = 2;
 983                         hpts->p_direct_wake = 1;
 984                         tcp_wakeinput(hpts);
 985                 }
 986         } else if (hpts->p_hpts_active == 0) {
 987                 retval = 4;
 988                 hpts->p_direct_wake = 1;
 989                 tcp_wakeinput(hpts);
 990         }
 991         return (retval);
 992 }
 993
 994 void
 995 tcp_queue_pkt_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
 996     int32_t tlen, int32_t drop_hdrlen, uint8_t iptos)
 997 {
 998         /* Setup packet for input first */
 999         INP_WLOCK_ASSERT(tp->t_inpcb);
1000         m->m_pkthdr.pace_thoff = (uint16_t) ((caddr_t)th - mtod(m, caddr_t));
1001         m->m_pkthdr.pace_tlen = (uint16_t) tlen;
1002         m->m_pkthdr.pace_drphdrlen = drop_hdrlen;
1003         m->m_pkthdr.pace_tos = iptos;
1004         m->m_pkthdr.pace_lock = (curthread->td_epochnest != 0);
1005         if (tp->t_in_pkt == NULL) {
1006                 tp->t_in_pkt = m;
1007                 tp->t_tail_pkt = m;
1008         } else {
1009                 tp->t_tail_pkt->m_nextpkt = m;
1010                 tp->t_tail_pkt = m;
1011         }
1012 }
1013
1014
1015 int32_t
1016 __tcp_queue_to_input(struct tcpcb *tp, struct mbuf *m, struct tcphdr *th,
1017     int32_t tlen, int32_t drop_hdrlen, uint8_t iptos, int32_t line){
1018         struct tcp_hpts_entry *hpts;
1019         int32_t ret;
1020
1021         tcp_queue_pkt_to_input(tp, m, th, tlen, drop_hdrlen, iptos);
1022         hpts = tcp_input_lock(tp->t_inpcb);
1023         ret = __tcp_queue_to_input_locked(tp->t_inpcb, hpts, line);
1024         mtx_unlock(&hpts->p_mtx);
1025         return (ret);
1026 }
1027
1028 void
1029 __tcp_set_inp_to_drop(struct inpcb *inp, uint16_t reason, int32_t line)
1030 {
1031         struct tcp_hpts_entry *hpts;
1032         struct tcpcb *tp;
1033
1034         tp = intotcpcb(inp);
1035         hpts = tcp_input_lock(tp->t_inpcb);
1036         if (inp->inp_in_input == 0) {
1037                 /* Ok we need to set it on the hpts in the current slot */
1038                 hpts_sane_input_insert(hpts, inp, line);
1039                 if (hpts->p_hpts_active == 0) {
1040                         /*
1041                          * Activate the hpts if it is sleeping.
1042                          */
1043                         hpts->p_direct_wake = 1;
1044                         tcp_wakeinput(hpts);
1045                 }
1046         } else if (hpts->p_hpts_active == 0) {
1047                 hpts->p_direct_wake = 1;
1048                 tcp_wakeinput(hpts);
1049         }
1050         inp->inp_hpts_drop_reas = reason;
1051         mtx_unlock(&hpts->p_mtx);
1052 }
1053
1054 static uint16_t
1055 hpts_random_cpu(struct inpcb *inp){
1056         /*
1057          * No flow type set distribute the load randomly.
1058          */
1059         uint16_t cpuid;
1060         uint32_t ran;
1061
1062         /*
1063          * If one has been set use it i.e. we want both in and out on the
1064          * same hpts.
1065          */
1066         if (inp->inp_input_cpu_set) {
1067                 return (inp->inp_input_cpu);
1068         } else if (inp->inp_hpts_cpu_set) {
1069                 return (inp->inp_hpts_cpu);
1070         }
1071         /* Nothing set use a random number */
1072         ran = arc4random();
1073         cpuid = (ran & 0xffff) % mp_ncpus;
1074         return (cpuid);
1075 }
1076
1077 static uint16_t
1078 hpts_cpuid(struct inpcb *inp){
1079         uint16_t cpuid;
1080
1081
1082         /*
1083          * If one has been set use it i.e. we want both in and out on the
1084          * same hpts.
1085          */
1086         if (inp->inp_input_cpu_set) {
1087                 return (inp->inp_input_cpu);
1088         } else if (inp->inp_hpts_cpu_set) {
1089                 return (inp->inp_hpts_cpu);
1090         }
1091         /* If one is set the other must be the same */
1092 #ifdef  RSS
1093         cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype);
1094         if (cpuid == NETISR_CPUID_NONE)
1095                 return (hpts_random_cpu(inp));
1096         else
1097                 return (cpuid);
1098 #else
1099         /*
1100          * We don't have a flowid -> cpuid mapping, so cheat and just map
1101          * unknown cpuids to curcpu.  Not the best, but apparently better
1102          * than defaulting to swi 0.
1103          */
1104         if (inp->inp_flowtype != M_HASHTYPE_NONE) {
1105                 cpuid = inp->inp_flowid % mp_ncpus;
1106                 return (cpuid);
1107         }
1108         cpuid = hpts_random_cpu(inp);
1109         return (cpuid);
1110 #endif
1111 }
1112
1113 /*
1114  * Do NOT try to optimize the processing of inp's
1115  * by first pulling off all the inp's into a temporary
1116  * list (e.g. TAILQ_CONCAT). If you do that the subtle
1117  * interactions of switching CPU's will kill because of
1118  * problems in the linked list manipulation. Basically
1119  * you would switch cpu's with the hpts mutex locked
1120  * but then while you were processing one of the inp's
1121  * some other one that you switch will get a new
1122  * packet on the different CPU. It will insert it
1123  * on the new hptss input list. Creating a temporary
1124  * link in the inp will not fix it either, since
1125  * the other hpts will be doing the same thing and
1126  * you will both end up using the temporary link.
1127  *
1128  * You will die in an ASSERT for tailq corruption if you
1129  * run INVARIANTS or you will die horribly without
1130  * INVARIANTS in some unknown way with a corrupt linked
1131  * list.
1132  */
1133 static void
1134 tcp_input_data(struct tcp_hpts_entry *hpts, struct timeval *tv)
1135 {
1136         struct mbuf *m, *n;
1137         struct tcpcb *tp;
1138         struct inpcb *inp;
1139         uint16_t drop_reason;
1140         int16_t set_cpu;
1141         uint32_t did_prefetch = 0;
1142         int32_t ti_locked = TI_UNLOCKED;
1143         struct epoch_tracker et;
1144
1145         HPTS_MTX_ASSERT(hpts);
1146         while ((inp = TAILQ_FIRST(&hpts->p_input)) != NULL) {
1147                 HPTS_MTX_ASSERT(hpts);
1148                 hpts_sane_input_remove(hpts, inp, 0);
1149                 if (inp->inp_input_cpu_set == 0) {
1150                         set_cpu = 1;
1151                 } else {
1152                         set_cpu = 0;
1153                 }
1154                 hpts->p_inp = inp;
1155                 drop_reason = inp->inp_hpts_drop_reas;
1156                 inp->inp_in_input = 0;
1157                 mtx_unlock(&hpts->p_mtx);
1158                 CURVNET_SET(inp->inp_vnet);
1159                 if (drop_reason) {
1160                         INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1161                         ti_locked = TI_RLOCKED;
1162                 } else {
1163                         ti_locked = TI_UNLOCKED;
1164                 }
1165                 INP_WLOCK(inp);
1166                 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
1167                     (inp->inp_flags2 & INP_FREED)) {
1168 out:
1169                         hpts->p_inp = NULL;
1170                         if (ti_locked == TI_RLOCKED) {
1171                                 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1172                         }
1173                         if (in_pcbrele_wlocked(inp) == 0) {
1174                                 INP_WUNLOCK(inp);
1175                         }
1176                         ti_locked = TI_UNLOCKED;
1177                         CURVNET_RESTORE();
1178                         mtx_lock(&hpts->p_mtx);
1179                         continue;
1180                 }
1181                 tp = intotcpcb(inp);
1182                 if ((tp == NULL) || (tp->t_inpcb == NULL)) {
1183                         goto out;
1184                 }
1185                 if (drop_reason) {
1186                         /* This tcb is being destroyed for drop_reason */
1187                         m = tp->t_in_pkt;
1188                         if (m)
1189                                 n = m->m_nextpkt;
1190                         else
1191                                 n = NULL;
1192                         tp->t_in_pkt = NULL;
1193                         while (m) {
1194                                 m_freem(m);
1195                                 m = n;
1196                                 if (m)
1197                                         n = m->m_nextpkt;
1198                         }
1199                         tp = tcp_drop(tp, drop_reason);
1200                         INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1201                         if (tp == NULL) {
1202                                 INP_WLOCK(inp);
1203                         }
1204                         if (in_pcbrele_wlocked(inp) == 0)
1205                                 INP_WUNLOCK(inp);
1206                         CURVNET_RESTORE();
1207                         mtx_lock(&hpts->p_mtx);
1208                         continue;
1209                 }
1210                 if (set_cpu) {
1211                         /*
1212                          * Setup so the next time we will move to the right
1213                          * CPU. This should be a rare event. It will
1214                          * sometimes happens when we are the client side
1215                          * (usually not the server). Somehow tcp_output()
1216                          * gets called before the tcp_do_segment() sets the
1217                          * intial state. This means the r_cpu and r_hpts_cpu
1218                          * is 0. We get on the hpts, and then tcp_input()
1219                          * gets called setting up the r_cpu to the correct
1220                          * value. The hpts goes off and sees the mis-match.
1221                          * We simply correct it here and the CPU will switch
1222                          * to the new hpts nextime the tcb gets added to the
1223                          * the hpts (not this time) :-)
1224                          */
1225                         tcp_set_hpts(inp);
1226                 }
1227                 m = tp->t_in_pkt;
1228                 n = NULL;
1229                 if (m != NULL &&
1230                     (m->m_pkthdr.pace_lock == TI_RLOCKED ||
1231                     tp->t_state != TCPS_ESTABLISHED)) {
1232                         ti_locked = TI_RLOCKED;
1233                         INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1234                         m = tp->t_in_pkt;
1235                 }
1236                 if (in_newts_every_tcb) {
1237                         if (in_ts_percision)
1238                                 microuptime(tv);
1239                         else
1240                                 getmicrouptime(tv);
1241                 }
1242                 if (tp->t_fb_ptr != NULL) {
1243                         kern_prefetch(tp->t_fb_ptr, &did_prefetch);
1244                         did_prefetch = 1;
1245                 }
1246                 /* Any input work to do, if so do it first */
1247                 if ((m != NULL) && (m == tp->t_in_pkt)) {
1248                         struct tcphdr *th;
1249                         int32_t tlen, drop_hdrlen, nxt_pkt;
1250                         uint8_t iptos;
1251
1252                         n = m->m_nextpkt;
1253                         tp->t_in_pkt = tp->t_tail_pkt = NULL;
1254                         while (m) {
1255                                 th = (struct tcphdr *)(mtod(m, caddr_t)+m->m_pkthdr.pace_thoff);
1256                                 tlen = m->m_pkthdr.pace_tlen;
1257                                 drop_hdrlen = m->m_pkthdr.pace_drphdrlen;
1258                                 iptos = m->m_pkthdr.pace_tos;
1259                                 m->m_nextpkt = NULL;
1260                                 if (n)
1261                                         nxt_pkt = 1;
1262                                 else
1263                                         nxt_pkt = 0;
1264                                 inp->inp_input_calls = 1;
1265                                 if (tp->t_fb->tfb_tcp_hpts_do_segment) {
1266                                         /* Use the hpts specific do_segment */
1267                                         (*tp->t_fb->tfb_tcp_hpts_do_segment) (m, th, inp->inp_socket,
1268                                             tp, drop_hdrlen,
1269                                             tlen, iptos, nxt_pkt, tv);
1270                                 } else {
1271                                         /* Use the default do_segment */
1272                                         (*tp->t_fb->tfb_tcp_do_segment) (m, th, inp->inp_socket,
1273                                             tp, drop_hdrlen,
1274                                                 tlen, iptos);
1275                                 }
1276                                 if (ti_locked == TI_RLOCKED)
1277                                         INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1278                                 /*
1279                                  * Do segment returns unlocked we need the
1280                                  * lock again but we also need some kasserts
1281                                  * here.
1282                                  */
1283                                 INP_INFO_UNLOCK_ASSERT(&V_tcbinfo);
1284                                 INP_UNLOCK_ASSERT(inp);
1285                                 m = n;
1286                                 if (m)
1287                                         n = m->m_nextpkt;
1288                                 if (m != NULL &&
1289                                     m->m_pkthdr.pace_lock == TI_RLOCKED) {
1290                                         INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1291                                         ti_locked = TI_RLOCKED;
1292                                 } else
1293                                         ti_locked = TI_UNLOCKED;
1294                                 INP_WLOCK(inp);
1295                                 /*
1296                                  * Since we have an opening here we must
1297                                  * re-check if the tcb went away while we
1298                                  * were getting the lock(s).
1299                                  */
1300                                 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) ||
1301                                     (inp->inp_flags2 & INP_FREED)) {
1302                                         while (m) {
1303                                                 m_freem(m);
1304                                                 m = n;
1305                                                 if (m)
1306                                                         n = m->m_nextpkt;
1307                                         }
1308                                         goto out;
1309                                 }
1310                                 /*
1311                                  * Now that we hold the INP lock, check if
1312                                  * we need to upgrade our lock.
1313                                  */
1314                                 if (ti_locked == TI_UNLOCKED &&
1315                                     (tp->t_state != TCPS_ESTABLISHED)) {
1316                                         ti_locked = TI_RLOCKED;
1317                                         INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1318                                 }
1319                         }       /** end while(m) */
1320                 }               /** end if ((m != NULL)  && (m == tp->t_in_pkt)) */
1321                 if (in_pcbrele_wlocked(inp) == 0)
1322                         INP_WUNLOCK(inp);
1323                 if (ti_locked == TI_RLOCKED)
1324                         INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1325                 INP_INFO_UNLOCK_ASSERT(&V_tcbinfo);
1326                 INP_UNLOCK_ASSERT(inp);
1327                 ti_locked = TI_UNLOCKED;
1328                 mtx_lock(&hpts->p_mtx);
1329                 hpts->p_inp = NULL;
1330                 CURVNET_RESTORE();
1331         }
1332 }
1333
1334 static int
1335 tcp_hpts_est_run(struct tcp_hpts_entry *hpts)
1336 {
1337         int32_t ticks_to_run;
1338
1339         if (hpts->p_prevtick && (SEQ_GT(hpts->p_curtick, hpts->p_prevtick))) {
1340                 ticks_to_run = hpts->p_curtick - hpts->p_prevtick;
1341                 if (ticks_to_run >= (NUM_OF_HPTSI_SLOTS - 1)) {
1342                         ticks_to_run = NUM_OF_HPTSI_SLOTS - 2;
1343                 }
1344         } else {
1345                 if (hpts->p_prevtick == hpts->p_curtick) {
1346                         /* This happens when we get woken up right away */
1347                         return (-1);
1348                 }
1349                 ticks_to_run = 1;
1350         }
1351         /* Set in where we will be when we catch up */
1352         hpts->p_nxt_slot = (hpts->p_cur_slot + ticks_to_run) % NUM_OF_HPTSI_SLOTS;
1353         if (hpts->p_nxt_slot == hpts->p_cur_slot) {
1354                 panic("Impossible math -- hpts:%p p_nxt_slot:%d p_cur_slot:%d ticks_to_run:%d",
1355                     hpts, hpts->p_nxt_slot, hpts->p_cur_slot, ticks_to_run);
1356         }
1357         return (ticks_to_run);
1358 }
1359
1360 static void
1361 tcp_hptsi(struct tcp_hpts_entry *hpts, struct timeval *ctick)
1362 {
1363         struct tcpcb *tp;
1364         struct inpcb *inp = NULL, *ninp;
1365         struct timeval tv;
1366         int32_t ticks_to_run, i, error, tick_now, interum_tick;
1367         int32_t paced_cnt = 0;
1368         int32_t did_prefetch = 0;
1369         int32_t prefetch_ninp = 0;
1370         int32_t prefetch_tp = 0;
1371         uint32_t cts;
1372         int16_t set_cpu;
1373
1374         HPTS_MTX_ASSERT(hpts);
1375         hpts->p_curtick = tcp_tv_to_hptstick(ctick);
1376         cts = tcp_tv_to_usectick(ctick);
1377         memcpy(&tv, ctick, sizeof(struct timeval));
1378         hpts->p_cur_slot = hpts_tick(hpts, 1);
1379
1380         /* Figure out if we had missed ticks */
1381 again:
1382         HPTS_MTX_ASSERT(hpts);
1383         ticks_to_run = tcp_hpts_est_run(hpts);
1384         if (!TAILQ_EMPTY(&hpts->p_input)) {
1385                 tcp_input_data(hpts, &tv);
1386         }
1387 #ifdef INVARIANTS
1388         if (TAILQ_EMPTY(&hpts->p_input) &&
1389             (hpts->p_on_inqueue_cnt != 0)) {
1390                 panic("tp:%p in_hpts input empty but cnt:%d",
1391                     hpts, hpts->p_on_inqueue_cnt);
1392         }
1393 #endif
1394         HPTS_MTX_ASSERT(hpts);
1395         /* Reset the ticks to run and time if we need too */
1396         interum_tick = tcp_gethptstick(&tv);
1397         if (interum_tick != hpts->p_curtick) {
1398                 /* Save off the new time we execute to */
1399                 *ctick = tv;
1400                 hpts->p_curtick = interum_tick;
1401                 cts = tcp_tv_to_usectick(&tv);
1402                 hpts->p_cur_slot = hpts_tick(hpts, 1);
1403                 ticks_to_run = tcp_hpts_est_run(hpts);
1404         }
1405         if (ticks_to_run == -1) {
1406                 goto no_run;
1407         }
1408         if (logging_on) {
1409                 tcp_hpts_log_it(hpts, inp, HPTSLOG_SETTORUN, ticks_to_run, 0);
1410         }
1411         if (hpts->p_on_queue_cnt == 0) {
1412                 goto no_one;
1413         }
1414         HPTS_MTX_ASSERT(hpts);
1415         for (i = 0; i < ticks_to_run; i++) {
1416                 /*
1417                  * Calculate our delay, if there are no extra ticks there
1418                  * was not any
1419                  */
1420                 hpts->p_delayed_by = (ticks_to_run - (i + 1)) * HPTS_TICKS_PER_USEC;
1421                 HPTS_MTX_ASSERT(hpts);
1422                 while ((inp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) {
1423                         /* For debugging */
1424                         if (logging_on) {
1425                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_HPTSI, ticks_to_run, i);
1426                         }
1427                         hpts->p_inp = inp;
1428                         paced_cnt++;
1429                         if (hpts->p_cur_slot != inp->inp_hptsslot) {
1430                                 panic("Hpts:%p inp:%p slot mis-aligned %u vs %u",
1431                                     hpts, inp, hpts->p_cur_slot, inp->inp_hptsslot);
1432                         }
1433                         /* Now pull it */
1434                         if (inp->inp_hpts_cpu_set == 0) {
1435                                 set_cpu = 1;
1436                         } else {
1437                                 set_cpu = 0;
1438                         }
1439                         hpts_sane_pace_remove(hpts, inp, &hpts->p_hptss[hpts->p_cur_slot], 0);
1440                         if ((ninp = TAILQ_FIRST(&hpts->p_hptss[hpts->p_cur_slot])) != NULL) {
1441                                 /* We prefetch the next inp if possible */
1442                                 kern_prefetch(ninp, &prefetch_ninp);
1443                                 prefetch_ninp = 1;
1444                         }
1445                         if (inp->inp_hpts_request) {
1446                                 /*
1447                                  * This guy is deferred out further in time
1448                                  * then our wheel had on it. Push him back
1449                                  * on the wheel.
1450                                  */
1451                                 int32_t remaining_slots;
1452
1453                                 remaining_slots = ticks_to_run - (i + 1);
1454                                 if (inp->inp_hpts_request > remaining_slots) {
1455                                         /*
1456                                          * Keep INVARIANTS happy by clearing
1457                                          * the flag
1458                                          */
1459                                         tcp_hpts_insert_locked(hpts, inp, inp->inp_hpts_request, cts, __LINE__, NULL, 1);
1460                                         hpts->p_inp = NULL;
1461                                         continue;
1462                                 }
1463                                 inp->inp_hpts_request = 0;
1464                         }
1465                         /*
1466                          * We clear the hpts flag here after dealing with
1467                          * remaining slots. This way anyone looking with the
1468                          * TCB lock will see its on the hpts until just
1469                          * before we unlock.
1470                          */
1471                         inp->inp_in_hpts = 0;
1472                         mtx_unlock(&hpts->p_mtx);
1473                         INP_WLOCK(inp);
1474                         if (in_pcbrele_wlocked(inp)) {
1475                                 mtx_lock(&hpts->p_mtx);
1476                                 if (logging_on)
1477                                         tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 1);
1478                                 hpts->p_inp = NULL;
1479                                 continue;
1480                         }
1481                         if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
1482 out_now:
1483 #ifdef INVARIANTS
1484                                 if (mtx_owned(&hpts->p_mtx)) {
1485                                         panic("Hpts:%p owns mtx prior-to lock line:%d",
1486                                             hpts, __LINE__);
1487                                 }
1488 #endif
1489                                 INP_WUNLOCK(inp);
1490                                 mtx_lock(&hpts->p_mtx);
1491                                 if (logging_on)
1492                                         tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 3);
1493                                 hpts->p_inp = NULL;
1494                                 continue;
1495                         }
1496                         tp = intotcpcb(inp);
1497                         if ((tp == NULL) || (tp->t_inpcb == NULL)) {
1498                                 goto out_now;
1499                         }
1500                         if (set_cpu) {
1501                                 /*
1502                                  * Setup so the next time we will move to
1503                                  * the right CPU. This should be a rare
1504                                  * event. It will sometimes happens when we
1505                                  * are the client side (usually not the
1506                                  * server). Somehow tcp_output() gets called
1507                                  * before the tcp_do_segment() sets the
1508                                  * intial state. This means the r_cpu and
1509                                  * r_hpts_cpu is 0. We get on the hpts, and
1510                                  * then tcp_input() gets called setting up
1511                                  * the r_cpu to the correct value. The hpts
1512                                  * goes off and sees the mis-match. We
1513                                  * simply correct it here and the CPU will
1514                                  * switch to the new hpts nextime the tcb
1515                                  * gets added to the the hpts (not this one)
1516                                  * :-)
1517                                  */
1518                                 tcp_set_hpts(inp);
1519                         }
1520                         if (out_newts_every_tcb) {
1521                                 struct timeval sv;
1522
1523                                 if (out_ts_percision)
1524                                         microuptime(&sv);
1525                                 else
1526                                         getmicrouptime(&sv);
1527                                 cts = tcp_tv_to_usectick(&sv);
1528                         }
1529                         CURVNET_SET(inp->inp_vnet);
1530                         /*
1531                          * There is a hole here, we get the refcnt on the
1532                          * inp so it will still be preserved but to make
1533                          * sure we can get the INP we need to hold the p_mtx
1534                          * above while we pull out the tp/inp,  as long as
1535                          * fini gets the lock first we are assured of having
1536                          * a sane INP we can lock and test.
1537                          */
1538 #ifdef INVARIANTS
1539                         if (mtx_owned(&hpts->p_mtx)) {
1540                                 panic("Hpts:%p owns mtx before tcp-output:%d",
1541                                     hpts, __LINE__);
1542                         }
1543 #endif
1544                         if (tp->t_fb_ptr != NULL) {
1545                                 kern_prefetch(tp->t_fb_ptr, &did_prefetch);
1546                                 did_prefetch = 1;
1547                         }
1548                         inp->inp_hpts_calls = 1;
1549                         if (tp->t_fb->tfb_tcp_output_wtime != NULL) {
1550                                 error = (*tp->t_fb->tfb_tcp_output_wtime) (tp, &tv);
1551                         } else {
1552                                 error = tp->t_fb->tfb_tcp_output(tp);
1553                         }
1554                         if (ninp && ninp->inp_ppcb) {
1555                                 /*
1556                                  * If we have a nxt inp, see if we can
1557                                  * prefetch its ppcb. Note this may seem
1558                                  * "risky" since we have no locks (other
1559                                  * than the previous inp) and there no
1560                                  * assurance that ninp was not pulled while
1561                                  * we were processing inp and freed. If this
1562                                  * occured it could mean that either:
1563                                  *
1564                                  * a) Its NULL (which is fine we won't go
1565                                  * here) <or> b) Its valid (which is cool we
1566                                  * will prefetch it) <or> c) The inp got
1567                                  * freed back to the slab which was
1568                                  * reallocated. Then the piece of memory was
1569                                  * re-used and something else (not an
1570                                  * address) is in inp_ppcb. If that occurs
1571                                  * we don't crash, but take a TLB shootdown
1572                                  * performance hit (same as if it was NULL
1573                                  * and we tried to pre-fetch it).
1574                                  *
1575                                  * Considering that the likelyhood of <c> is
1576                                  * quite rare we will take a risk on doing
1577                                  * this. If performance drops after testing
1578                                  * we can always take this out. NB: the
1579                                  * kern_prefetch on amd64 actually has
1580                                  * protection against a bad address now via
1581                                  * the DMAP_() tests. This will prevent the
1582                                  * TLB hit, and instead if <c> occurs just
1583                                  * cause us to load cache with a useless
1584                                  * address (to us).
1585                                  */
1586                                 kern_prefetch(ninp->inp_ppcb, &prefetch_tp);
1587                                 prefetch_tp = 1;
1588                         }
1589                         INP_WUNLOCK(inp);
1590                         INP_UNLOCK_ASSERT(inp);
1591                         CURVNET_RESTORE();
1592 #ifdef INVARIANTS
1593                         if (mtx_owned(&hpts->p_mtx)) {
1594                                 panic("Hpts:%p owns mtx prior-to lock line:%d",
1595                                     hpts, __LINE__);
1596                         }
1597 #endif
1598                         mtx_lock(&hpts->p_mtx);
1599                         if (logging_on)
1600                                 tcp_hpts_log_it(hpts, hpts->p_inp, HPTSLOG_INP_DONE, 0, 4);
1601                         hpts->p_inp = NULL;
1602                 }
1603                 HPTS_MTX_ASSERT(hpts);
1604                 hpts->p_inp = NULL;
1605                 hpts->p_cur_slot++;
1606                 if (hpts->p_cur_slot >= NUM_OF_HPTSI_SLOTS) {
1607                         hpts->p_cur_slot = 0;
1608                 }
1609         }
1610 no_one:
1611         HPTS_MTX_ASSERT(hpts);
1612         hpts->p_prevtick = hpts->p_curtick;
1613         hpts->p_delayed_by = 0;
1614         /*
1615          * Check to see if we took an excess amount of time and need to run
1616          * more ticks (if we did not hit eno-bufs).
1617          */
1618         /* Re-run any input that may be there */
1619         (void)tcp_gethptstick(&tv);
1620         if (!TAILQ_EMPTY(&hpts->p_input)) {
1621                 tcp_input_data(hpts, &tv);
1622         }
1623 #ifdef INVARIANTS
1624         if (TAILQ_EMPTY(&hpts->p_input) &&
1625             (hpts->p_on_inqueue_cnt != 0)) {
1626                 panic("tp:%p in_hpts input empty but cnt:%d",
1627                     hpts, hpts->p_on_inqueue_cnt);
1628         }
1629 #endif
1630         tick_now = tcp_gethptstick(&tv);
1631         if (SEQ_GT(tick_now, hpts->p_prevtick)) {
1632                 struct timeval res;
1633
1634                 /* Did we really spend a full tick or more in here? */
1635                 timersub(&tv, ctick, &res);
1636                 if (res.tv_sec || (res.tv_usec >= HPTS_TICKS_PER_USEC)) {
1637                         counter_u64_add(hpts_loops, 1);
1638                         if (logging_on) {
1639                                 tcp_hpts_log_it(hpts, inp, HPTSLOG_TOLONG, (uint32_t) res.tv_usec, tick_now);
1640                         }
1641                         *ctick = res;
1642                         hpts->p_curtick = tick_now;
1643                         goto again;
1644                 }
1645         }
1646 no_run:
1647         {
1648                 uint32_t t = 0, i, fnd = 0;
1649
1650                 if (hpts->p_on_queue_cnt) {
1651
1652
1653                         /*
1654                          * Find next slot that is occupied and use that to
1655                          * be the sleep time.
1656                          */
1657                         for (i = 1, t = hpts->p_nxt_slot; i < NUM_OF_HPTSI_SLOTS; i++) {
1658                                 if (TAILQ_EMPTY(&hpts->p_hptss[t]) == 0) {
1659                                         fnd = 1;
1660                                         break;
1661                                 }
1662                                 t = (t + 1) % NUM_OF_HPTSI_SLOTS;
1663                         }
1664                         if (fnd) {
1665                                 hpts->p_hpts_sleep_time = i;
1666                         } else {
1667                                 counter_u64_add(back_tosleep, 1);
1668 #ifdef INVARIANTS
1669                                 panic("Hpts:%p cnt:%d but non found", hpts, hpts->p_on_queue_cnt);
1670 #endif
1671                                 hpts->p_on_queue_cnt = 0;
1672                                 goto non_found;
1673                         }
1674                         t++;
1675                 } else {
1676                         /* No one on the wheel sleep for all but 2 slots  */
1677 non_found:
1678                         if (hpts_sleep_max == 0)
1679                                 hpts_sleep_max = 1;
1680                         hpts->p_hpts_sleep_time = min((NUM_OF_HPTSI_SLOTS - 2), hpts_sleep_max);
1681                         t = 0;
1682                 }
1683                 if (logging_on) {
1684                         tcp_hpts_log_it(hpts, inp, HPTSLOG_SLEEPSET, t, (hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC));
1685                 }
1686         }
1687 }
1688
1689 void
1690 __tcp_set_hpts(struct inpcb *inp, int32_t line)
1691 {
1692         struct tcp_hpts_entry *hpts;
1693
1694         INP_WLOCK_ASSERT(inp);
1695         hpts = tcp_hpts_lock(inp);
1696         if ((inp->inp_in_hpts == 0) &&
1697             (inp->inp_hpts_cpu_set == 0)) {
1698                 inp->inp_hpts_cpu = hpts_cpuid(inp);
1699                 inp->inp_hpts_cpu_set = 1;
1700         }
1701         mtx_unlock(&hpts->p_mtx);
1702         hpts = tcp_input_lock(inp);
1703         if ((inp->inp_input_cpu_set == 0) &&
1704             (inp->inp_in_input == 0)) {
1705                 inp->inp_input_cpu = hpts_cpuid(inp);
1706                 inp->inp_input_cpu_set = 1;
1707         }
1708         mtx_unlock(&hpts->p_mtx);
1709 }
1710
1711 uint16_t
1712 tcp_hpts_delayedby(struct inpcb *inp){
1713         return (tcp_pace.rp_ent[inp->inp_hpts_cpu]->p_delayed_by);
1714 }
1715
1716 static void
1717 tcp_hpts_thread(void *ctx)
1718 {
1719         struct tcp_hpts_entry *hpts;
1720         struct timeval tv;
1721         sbintime_t sb;
1722
1723         hpts = (struct tcp_hpts_entry *)ctx;
1724         mtx_lock(&hpts->p_mtx);
1725         if (hpts->p_direct_wake) {
1726                 /* Signaled by input */
1727                 if (logging_on)
1728                         tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 1, 1);
1729                 callout_stop(&hpts->co);
1730         } else {
1731                 /* Timed out */
1732                 if (callout_pending(&hpts->co) ||
1733                     !callout_active(&hpts->co)) {
1734                         if (logging_on)
1735                                 tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 2, 2);
1736                         mtx_unlock(&hpts->p_mtx);
1737                         return;
1738                 }
1739                 callout_deactivate(&hpts->co);
1740                 if (logging_on)
1741                         tcp_hpts_log_it(hpts, NULL, HPTSLOG_AWAKE, 3, 3);
1742         }
1743         hpts->p_hpts_active = 1;
1744         (void)tcp_gethptstick(&tv);
1745         tcp_hptsi(hpts, &tv);
1746         HPTS_MTX_ASSERT(hpts);
1747         tv.tv_sec = 0;
1748         tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC;
1749         if (tcp_min_hptsi_time && (tv.tv_usec < tcp_min_hptsi_time)) {
1750                 tv.tv_usec = tcp_min_hptsi_time;
1751                 hpts->p_on_min_sleep = 1;
1752         } else {
1753                 /* Clear the min sleep flag */
1754                 hpts->p_on_min_sleep = 0;
1755         }
1756         hpts->p_hpts_active = 0;
1757         sb = tvtosbt(tv);
1758         if (tcp_hpts_callout_skip_swi == 0) {
1759                 callout_reset_sbt_on(&hpts->co, sb, 0,
1760                     hpts_timeout_swi, hpts, hpts->p_cpu,
1761                     (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
1762         } else {
1763                 callout_reset_sbt_on(&hpts->co, sb, 0,
1764                     hpts_timeout_dir, hpts,
1765                     hpts->p_cpu,
1766                     C_PREL(tcp_hpts_precision));
1767         }
1768         hpts->p_direct_wake = 0;
1769         mtx_unlock(&hpts->p_mtx);
1770 }
1771
1772 #undef  timersub
1773
1774 static void
1775 tcp_init_hptsi(void *st)
1776 {
1777         int32_t i, j, error, bound = 0, created = 0;
1778         size_t sz, asz;
1779         struct timeval tv;
1780         sbintime_t sb;
1781         struct tcp_hpts_entry *hpts;
1782         char unit[16];
1783         uint32_t ncpus = mp_ncpus ? mp_ncpus : MAXCPU;
1784
1785         tcp_pace.rp_proc = NULL;
1786         tcp_pace.rp_num_hptss = ncpus;
1787         hpts_loops = counter_u64_alloc(M_WAITOK);
1788         back_tosleep = counter_u64_alloc(M_WAITOK);
1789
1790         sz = (tcp_pace.rp_num_hptss * sizeof(struct tcp_hpts_entry *));
1791         tcp_pace.rp_ent = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO);
1792         asz = sizeof(struct hptsh) * NUM_OF_HPTSI_SLOTS;
1793         for (i = 0; i < tcp_pace.rp_num_hptss; i++) {
1794                 tcp_pace.rp_ent[i] = malloc(sizeof(struct tcp_hpts_entry),
1795                     M_TCPHPTS, M_WAITOK | M_ZERO);
1796                 tcp_pace.rp_ent[i]->p_hptss = malloc(asz,
1797                     M_TCPHPTS, M_WAITOK);
1798                 hpts = tcp_pace.rp_ent[i];
1799                 /*
1800                  * Init all the hpts structures that are not specifically
1801                  * zero'd by the allocations. Also lets attach them to the
1802                  * appropriate sysctl block as well.
1803                  */
1804                 mtx_init(&hpts->p_mtx, "tcp_hpts_lck",
1805                     "hpts", MTX_DEF | MTX_DUPOK);
1806                 TAILQ_INIT(&hpts->p_input);
1807                 for (j = 0; j < NUM_OF_HPTSI_SLOTS; j++) {
1808                         TAILQ_INIT(&hpts->p_hptss[j]);
1809                 }
1810                 sysctl_ctx_init(&hpts->hpts_ctx);
1811                 sprintf(unit, "%d", i);
1812                 hpts->hpts_root = SYSCTL_ADD_NODE(&hpts->hpts_ctx,
1813                     SYSCTL_STATIC_CHILDREN(_net_inet_tcp_hpts),
1814                     OID_AUTO,
1815                     unit,
1816                     CTLFLAG_RW, 0,
1817                     "");
1818                 SYSCTL_ADD_INT(&hpts->hpts_ctx,
1819                     SYSCTL_CHILDREN(hpts->hpts_root),
1820                     OID_AUTO, "in_qcnt", CTLFLAG_RD,
1821                     &hpts->p_on_inqueue_cnt, 0,
1822                     "Count TCB's awaiting input processing");
1823                 SYSCTL_ADD_INT(&hpts->hpts_ctx,
1824                     SYSCTL_CHILDREN(hpts->hpts_root),
1825                     OID_AUTO, "out_qcnt", CTLFLAG_RD,
1826                     &hpts->p_on_queue_cnt, 0,
1827                     "Count TCB's awaiting output processing");
1828                 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1829                     SYSCTL_CHILDREN(hpts->hpts_root),
1830                     OID_AUTO, "active", CTLFLAG_RD,
1831                     &hpts->p_hpts_active, 0,
1832                     "Is the hpts active");
1833                 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1834                     SYSCTL_CHILDREN(hpts->hpts_root),
1835                     OID_AUTO, "curslot", CTLFLAG_RD,
1836                     &hpts->p_cur_slot, 0,
1837                     "What the current slot is if active");
1838                 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1839                     SYSCTL_CHILDREN(hpts->hpts_root),
1840                     OID_AUTO, "curtick", CTLFLAG_RD,
1841                     &hpts->p_curtick, 0,
1842                     "What the current tick on if active");
1843                 SYSCTL_ADD_UINT(&hpts->hpts_ctx,
1844                     SYSCTL_CHILDREN(hpts->hpts_root),
1845                     OID_AUTO, "logsize", CTLFLAG_RD,
1846                     &hpts->p_logsize, 0,
1847                     "Hpts logging buffer size");
1848                 hpts->p_hpts_sleep_time = NUM_OF_HPTSI_SLOTS - 2;
1849                 hpts->p_num = i;
1850                 hpts->p_prevtick = hpts->p_curtick = tcp_gethptstick(&tv);
1851                 hpts->p_prevtick -= 1;
1852                 hpts->p_prevtick %= NUM_OF_HPTSI_SLOTS;
1853                 hpts->p_cpu = 0xffff;
1854                 hpts->p_nxt_slot = 1;
1855                 hpts->p_logsize = tcp_hpts_logging_size;
1856                 if (hpts->p_logsize) {
1857                         sz = (sizeof(struct hpts_log) * hpts->p_logsize);
1858                         hpts->p_log = malloc(sz, M_TCPHPTS, M_WAITOK | M_ZERO);
1859                 }
1860                 callout_init(&hpts->co, 1);
1861         }
1862         /*
1863          * Now lets start ithreads to handle the hptss.
1864          */
1865         CPU_FOREACH(i) {
1866                 hpts = tcp_pace.rp_ent[i];
1867                 hpts->p_cpu = i;
1868                 error = swi_add(&hpts->ie, "hpts",
1869                     tcp_hpts_thread, (void *)hpts,
1870                     SWI_NET, INTR_MPSAFE, &hpts->ie_cookie);
1871                 if (error) {
1872                         panic("Can't add hpts:%p i:%d err:%d",
1873                             hpts, i, error);
1874                 }
1875                 created++;
1876                 if (tcp_bind_threads) {
1877                         if (intr_event_bind(hpts->ie, i) == 0)
1878                                 bound++;
1879                 }
1880                 tv.tv_sec = 0;
1881                 tv.tv_usec = hpts->p_hpts_sleep_time * HPTS_TICKS_PER_USEC;
1882                 sb = tvtosbt(tv);
1883                 if (tcp_hpts_callout_skip_swi == 0) {
1884                         callout_reset_sbt_on(&hpts->co, sb, 0,
1885                             hpts_timeout_swi, hpts, hpts->p_cpu,
1886                             (C_DIRECT_EXEC | C_PREL(tcp_hpts_precision)));
1887                 } else {
1888                         callout_reset_sbt_on(&hpts->co, sb, 0,
1889                             hpts_timeout_dir, hpts,
1890                             hpts->p_cpu,
1891                             C_PREL(tcp_hpts_precision));
1892                 }
1893         }
1894         printf("TCP Hpts created %d swi interrupt thread and bound %d\n",
1895             created, bound);
1896         return;
1897 }
1898
1899 SYSINIT(tcphptsi, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, tcp_init_hptsi, NULL);
1900 MODULE_VERSION(tcphpts, 1);