sys/netinet/tcp_reass.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1994, 1995
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. Neither the name of the University nor the names of its contributors
  16  *    may be used to endorse or promote products derived from this software
  17  *    without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  *      @(#)tcp_input.c 8.12 (Berkeley) 5/24/95
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_inet.h"
  38 #include "opt_inet6.h"
  39 #include "opt_tcpdebug.h"
  40
  41 #include <sys/param.h>
  42 #include <sys/kernel.h>
  43 #include <sys/eventhandler.h>
  44 #include <sys/malloc.h>
  45 #include <sys/mbuf.h>
  46 #include <sys/socket.h>
  47 #include <sys/socketvar.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/syslog.h>
  50 #include <sys/systm.h>
  51
  52 #include <vm/uma.h>
  53
  54 #include <net/if.h>
  55 #include <net/if_var.h>
  56 #include <net/route.h>
  57 #include <net/vnet.h>
  58
  59 #include <netinet/in.h>
  60 #include <netinet/in_pcb.h>
  61 #include <netinet/in_systm.h>
  62 #include <netinet/in_var.h>
  63 #include <netinet/ip.h>
  64 #include <netinet/ip_var.h>
  65 #include <netinet/ip_options.h>
  66 #include <netinet/ip6.h>
  67 #include <netinet6/in6_pcb.h>
  68 #include <netinet6/ip6_var.h>
  69 #include <netinet6/nd6.h>
  70 #include <netinet/tcp.h>
  71 #include <netinet/tcp_fsm.h>
  72 #include <netinet/tcp_seq.h>
  73 #include <netinet/tcp_timer.h>
  74 #include <netinet/tcp_var.h>
  75 #include <netinet/tcp_log_buf.h>
  76 #include <netinet/tcp_hpts.h>
  77 #include <netinet6/tcp6_var.h>
  78 #include <netinet/tcpip.h>
  79 #ifdef TCPDEBUG
  80 #include <netinet/tcp_debug.h>
  81 #endif /* TCPDEBUG */
  82
  83 #define TCP_R_LOG_ADD           1
  84 #define TCP_R_LOG_LIMIT_REACHED 2
  85 #define TCP_R_LOG_APPEND        3
  86 #define TCP_R_LOG_PREPEND       4
  87 #define TCP_R_LOG_REPLACE       5
  88 #define TCP_R_LOG_MERGE_INTO    6
  89 #define TCP_R_LOG_NEW_ENTRY     7
  90 #define TCP_R_LOG_READ          8
  91 #define TCP_R_LOG_ZERO          9
  92 #define TCP_R_LOG_DUMP          10
  93 #define TCP_R_LOG_TRIM          11
  94
  95 /* For debugging we want counters and BB logging */
  96 /* #define TCP_REASS_COUNTERS 1 */
  97 /* #define TCP_REASS_LOGGING 1 */
  98
  99 static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, reass, CTLFLAG_RW, 0,
 100     "TCP Segment Reassembly Queue");
 101
 102 static SYSCTL_NODE(_net_inet_tcp_reass, OID_AUTO, stats, CTLFLAG_RW, 0,
 103     "TCP Segment Reassembly stats");
 104
 105
 106 static int tcp_reass_maxseg = 0;
 107 SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, maxsegments, CTLFLAG_RDTUN,
 108     &tcp_reass_maxseg, 0,
 109     "Global maximum number of TCP Segments in Reassembly Queue");
 110
 111 static uma_zone_t tcp_reass_zone;
 112 SYSCTL_UMA_CUR(_net_inet_tcp_reass, OID_AUTO, cursegments, 0,
 113     &tcp_reass_zone,
 114     "Global number of TCP Segments currently in Reassembly Queue");
 115
 116 static u_int tcp_reass_maxqueuelen = 100;
 117 SYSCTL_UINT(_net_inet_tcp_reass, OID_AUTO, maxqueuelen, CTLFLAG_RWTUN,
 118     &tcp_reass_maxqueuelen, 0,
 119     "Maximum number of TCP Segments per Reassembly Queue");
 120
 121 static int tcp_new_limits = 0;
 122 SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, new_limit, CTLFLAG_RWTUN,
 123     &tcp_new_limits, 0,
 124     "Do we use the new limit method we are discussing?");
 125
 126 static u_int tcp_reass_queue_guard = 16;
 127 SYSCTL_UINT(_net_inet_tcp_reass, OID_AUTO, queueguard, CTLFLAG_RWTUN,
 128     &tcp_reass_queue_guard, 16,
 129     "Number of TCP Segments in Reassembly Queue where we flip over to guard mode");
 130
 131 #ifdef TCP_REASS_COUNTERS
 132
 133 counter_u64_t reass_entry;
 134 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, entry, CTLFLAG_RD,
 135     &reass_entry, "A segment entered reassembly ");
 136
 137 counter_u64_t reass_path1;
 138 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path1, CTLFLAG_RD,
 139     &reass_path1, "Took path 1");
 140
 141 counter_u64_t reass_path2;
 142 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path2, CTLFLAG_RD,
 143     &reass_path2, "Took path 2");
 144
 145 counter_u64_t reass_path3;
 146 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path3, CTLFLAG_RD,
 147     &reass_path3, "Took path 3");
 148
 149 counter_u64_t reass_path4;
 150 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path4, CTLFLAG_RD,
 151     &reass_path4, "Took path 4");
 152
 153 counter_u64_t reass_path5;
 154 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path5, CTLFLAG_RD,
 155     &reass_path5, "Took path 5");
 156
 157 counter_u64_t reass_path6;
 158 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path6, CTLFLAG_RD,
 159     &reass_path6, "Took path 6");
 160
 161 counter_u64_t reass_path7;
 162 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path7, CTLFLAG_RD,
 163     &reass_path7, "Took path 7");
 164
 165 counter_u64_t reass_fullwalk;
 166 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, fullwalk, CTLFLAG_RD,
 167     &reass_fullwalk, "Took a full walk ");
 168
 169 counter_u64_t reass_nospace;
 170 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, nospace, CTLFLAG_RD,
 171     &reass_nospace, "Had no mbuf capacity ");
 172
 173 counter_u64_t merge_fwd;
 174 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, merge_fwd, CTLFLAG_RD,
 175     &merge_fwd, "Ran merge fwd");
 176
 177 counter_u64_t merge_into;
 178 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, merge_into, CTLFLAG_RD,
 179     &merge_into, "Ran merge into");
 180
 181 counter_u64_t tcp_zero_input;
 182 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, zero_input, CTLFLAG_RD,
 183     &tcp_zero_input, "The reassembly buffer saw a zero len segment etc");
 184
 185 #endif
 186
 187 /* Initialize TCP reassembly queue */
 188 static void
 189 tcp_reass_zone_change(void *tag)
 190 {
 191
 192         /* Set the zone limit and read back the effective value. */
 193         tcp_reass_maxseg = nmbclusters / 16;
 194         tcp_reass_maxseg = uma_zone_set_max(tcp_reass_zone,
 195             tcp_reass_maxseg);
 196 }
 197
 198 #ifdef TCP_REASS_LOGGING
 199
 200 static void
 201 tcp_log_reassm(struct tcpcb *tp, struct tseg_qent *q, struct tseg_qent *p,
 202     tcp_seq seq, int len, uint8_t action, int instance)
 203 {
 204         uint32_t cts;
 205         struct timeval tv;
 206
 207         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
 208                 union tcp_log_stackspecific log;
 209
 210                 memset(&log, 0, sizeof(log));
 211                 cts = tcp_get_usecs(&tv);
 212                 log.u_bbr.flex1 = seq;
 213                 log.u_bbr.cur_del_rate = (uint64_t)q;
 214                 log.u_bbr.delRate = (uint64_t)p;
 215                 if (q != NULL) {
 216                         log.u_bbr.flex2 = q->tqe_start;
 217                         log.u_bbr.flex3 = q->tqe_len;
 218                         log.u_bbr.flex4 = q->tqe_mbuf_cnt;
 219                         log.u_bbr.hptsi_gain = q->tqe_flags;
 220                 }
 221                 if (p != NULL)  {
 222                         log.u_bbr.flex5 = p->tqe_start;
 223                         log.u_bbr.pkts_out = p->tqe_len;
 224                         log.u_bbr.epoch = p->tqe_mbuf_cnt;
 225                         log.u_bbr.cwnd_gain = p->tqe_flags;
 226                 }
 227                 log.u_bbr.flex6 = tp->t_segqmbuflen;
 228                 log.u_bbr.flex7 = instance;
 229                 log.u_bbr.flex8 = action;
 230                 log.u_bbr.timeStamp = cts;
 231                 TCP_LOG_EVENTP(tp, NULL,
 232                     &tp->t_inpcb->inp_socket->so_rcv,
 233                     &tp->t_inpcb->inp_socket->so_snd,
 234                     TCP_LOG_REASS, 0,
 235                     len, &log, false, &tv);
 236         }
 237 }
 238
 239 static void
 240 tcp_reass_log_dump(struct tcpcb *tp)
 241 {
 242         struct tseg_qent *q;
 243
 244         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
 245                 TAILQ_FOREACH(q, &tp->t_segq, tqe_q) {
 246                         tcp_log_reassm(tp, q, NULL, q->tqe_start, q->tqe_len, TCP_R_LOG_DUMP, 0);
 247                 }
 248         };
 249 }
 250
 251 static void
 252 tcp_reass_log_new_in(struct tcpcb *tp, tcp_seq seq, int len, struct mbuf *m,
 253     int logval, struct tseg_qent *q)
 254 {
 255         int cnt;
 256         struct mbuf *t;
 257
 258         cnt = 0;
 259         t = m;
 260         while (t) {
 261                 cnt += t->m_len;
 262                 t = t->m_next;
 263         }
 264         tcp_log_reassm(tp, q, NULL, seq, len, logval, cnt);
 265 }
 266
 267 #endif
 268
 269 void
 270 tcp_reass_global_init(void)
 271 {
 272
 273         tcp_reass_maxseg = nmbclusters / 16;
 274         TUNABLE_INT_FETCH("net.inet.tcp.reass.maxsegments",
 275             &tcp_reass_maxseg);
 276         tcp_reass_zone = uma_zcreate("tcpreass", sizeof (struct tseg_qent),
 277             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 278         /* Set the zone limit and read back the effective value. */
 279         tcp_reass_maxseg = uma_zone_set_max(tcp_reass_zone,
 280             tcp_reass_maxseg);
 281 #ifdef TCP_REASS_COUNTERS
 282         reass_path1 = counter_u64_alloc(M_WAITOK);
 283         reass_path2 = counter_u64_alloc(M_WAITOK);
 284         reass_path3 = counter_u64_alloc(M_WAITOK);
 285         reass_path4 = counter_u64_alloc(M_WAITOK);
 286         reass_path5 = counter_u64_alloc(M_WAITOK);
 287         reass_path6 = counter_u64_alloc(M_WAITOK);
 288         reass_path7 = counter_u64_alloc(M_WAITOK);
 289         reass_fullwalk = counter_u64_alloc(M_WAITOK);
 290         reass_nospace = counter_u64_alloc(M_WAITOK);
 291         reass_entry = counter_u64_alloc(M_WAITOK);
 292         merge_fwd = counter_u64_alloc(M_WAITOK);
 293         merge_into = counter_u64_alloc(M_WAITOK);
 294         tcp_zero_input = counter_u64_alloc(M_WAITOK);
 295 #endif
 296         EVENTHANDLER_REGISTER(nmbclusters_change,
 297             tcp_reass_zone_change, NULL, EVENTHANDLER_PRI_ANY);
 298
 299 }
 300
 301 void
 302 tcp_reass_flush(struct tcpcb *tp)
 303 {
 304         struct tseg_qent *qe;
 305
 306         INP_WLOCK_ASSERT(tp->t_inpcb);
 307
 308         while ((qe = TAILQ_FIRST(&tp->t_segq)) != NULL) {
 309                 TAILQ_REMOVE(&tp->t_segq, qe, tqe_q);
 310                 m_freem(qe->tqe_m);
 311                 uma_zfree(tcp_reass_zone, qe);
 312                 tp->t_segqlen--;
 313         }
 314         tp->t_segqmbuflen = 0;
 315         KASSERT((tp->t_segqlen == 0),
 316             ("TCP reass queue %p segment count is %d instead of 0 after flush.",
 317             tp, tp->t_segqlen));
 318 }
 319
 320 static void
 321 tcp_reass_append(struct tcpcb *tp, struct tseg_qent *last,
 322     struct mbuf *m, struct tcphdr *th, int tlen,
 323     struct mbuf *mlast, int lenofoh)
 324 {
 325
 326 #ifdef TCP_REASS_LOGGING
 327         tcp_log_reassm(tp, last, NULL, th->th_seq, tlen, TCP_R_LOG_APPEND, 0);
 328 #endif
 329         last->tqe_len += tlen;
 330         last->tqe_m->m_pkthdr.len += tlen;
 331         /* Preserve the FIN bit if its there */
 332         last->tqe_flags |= (th->th_flags & TH_FIN);
 333         last->tqe_last->m_next = m;
 334         last->tqe_last = mlast;
 335         last->tqe_mbuf_cnt += lenofoh;
 336         tp->t_rcvoopack++;
 337         TCPSTAT_INC(tcps_rcvoopack);
 338         TCPSTAT_ADD(tcps_rcvoobyte, tlen);
 339 #ifdef TCP_REASS_LOGGING
 340         tcp_reass_log_new_in(tp, last->tqe_start, lenofoh, last->tqe_m,
 341                              TCP_R_LOG_APPEND,
 342                              last);
 343 #endif
 344 }
 345
 346 static void
 347 tcp_reass_prepend(struct tcpcb *tp, struct tseg_qent *first, struct mbuf *m, struct tcphdr *th,
 348                   int tlen, struct mbuf *mlast, int lenofoh)
 349 {
 350         int i;
 351
 352 #ifdef TCP_REASS_LOGGING
 353         tcp_log_reassm(tp, first, NULL, th->th_seq, tlen, TCP_R_LOG_PREPEND, 0);
 354 #endif
 355         if (SEQ_GT((th->th_seq + tlen), first->tqe_start)) {
 356                 /* The new data overlaps into the old */
 357                 i = (th->th_seq + tlen) - first->tqe_start;
 358 #ifdef TCP_REASS_LOGGING
 359                 tcp_log_reassm(tp, first, NULL, 0, i, TCP_R_LOG_TRIM, 1);
 360 #endif
 361                 m_adj(first->tqe_m, i);
 362                 first->tqe_len -= i;
 363                 first->tqe_start += i;
 364         }
 365         /* Ok now setup our chain to point to the old first */
 366         mlast->m_next = first->tqe_m;
 367         first->tqe_m = m;
 368         first->tqe_len += tlen;
 369         first->tqe_start = th->th_seq;
 370         first->tqe_m->m_pkthdr.len = first->tqe_len;
 371         first->tqe_mbuf_cnt += lenofoh;
 372         tp->t_rcvoopack++;
 373         TCPSTAT_INC(tcps_rcvoopack);
 374         TCPSTAT_ADD(tcps_rcvoobyte, tlen);
 375 #ifdef TCP_REASS_LOGGING
 376         tcp_reass_log_new_in(tp, first->tqe_start, lenofoh, first->tqe_m,
 377                              TCP_R_LOG_PREPEND,
 378                              first);
 379 #endif
 380 }
 381
 382 static void
 383 tcp_reass_replace(struct tcpcb *tp, struct tseg_qent *q, struct mbuf *m,
 384     tcp_seq seq, int len, struct mbuf *mlast, int mbufoh, uint8_t flags)
 385 {
 386         /*
 387          * Free the data in q, and replace
 388          * it with the new segment.
 389          */
 390         int len_dif;
 391
 392 #ifdef TCP_REASS_LOGGING
 393         tcp_log_reassm(tp, q, NULL, seq, len, TCP_R_LOG_REPLACE, 0);
 394 #endif
 395         m_freem(q->tqe_m);
 396         KASSERT(tp->t_segqmbuflen >= q->tqe_mbuf_cnt,
 397                 ("Tp:%p seg queue goes negative", tp));
 398         tp->t_segqmbuflen -= q->tqe_mbuf_cnt;
 399         q->tqe_mbuf_cnt = mbufoh;
 400         q->tqe_m = m;
 401         q->tqe_last = mlast;
 402         q->tqe_start = seq;
 403         if (len > q->tqe_len)
 404                 len_dif = len - q->tqe_len;
 405         else
 406                 len_dif = 0;
 407         tp->t_rcvoopack++;
 408         TCPSTAT_INC(tcps_rcvoopack);
 409         TCPSTAT_ADD(tcps_rcvoobyte, len_dif);
 410         q->tqe_len = len;
 411         q->tqe_flags = (flags & TH_FIN);
 412         q->tqe_m->m_pkthdr.len = q->tqe_len;
 413         tp->t_segqmbuflen += mbufoh;
 414
 415 }
 416
 417 static void
 418 tcp_reass_merge_into(struct tcpcb *tp, struct tseg_qent *ent,
 419     struct tseg_qent *q)
 420 {
 421         /*
 422          * Merge q into ent and free q from the list.
 423          */
 424 #ifdef TCP_REASS_LOGGING
 425         tcp_log_reassm(tp, q, ent, 0, 0, TCP_R_LOG_MERGE_INTO, 0);
 426 #endif
 427 #ifdef TCP_REASS_COUNTERS
 428         counter_u64_add(merge_into, 1);
 429 #endif
 430         ent->tqe_last->m_next = q->tqe_m;
 431         ent->tqe_last = q->tqe_last;
 432         ent->tqe_len += q->tqe_len;
 433         ent->tqe_mbuf_cnt += q->tqe_mbuf_cnt;
 434         ent->tqe_m->m_pkthdr.len += q->tqe_len;
 435         ent->tqe_flags |= (q->tqe_flags & TH_FIN);
 436         TAILQ_REMOVE(&tp->t_segq, q, tqe_q);
 437         uma_zfree(tcp_reass_zone, q);
 438         tp->t_segqlen--;
 439
 440 }
 441
 442 static void
 443 tcp_reass_merge_forward(struct tcpcb *tp, struct tseg_qent *ent)
 444 {
 445         struct tseg_qent *q, *qtmp;
 446         int i;
 447         tcp_seq max;
 448         /*
 449          * Given an entry merge forward anyplace
 450          * that ent overlaps forward.
 451          */
 452
 453         max = ent->tqe_start + ent->tqe_len;
 454         q = TAILQ_NEXT(ent, tqe_q);
 455         if (q == NULL) {
 456                 /* Nothing left */
 457                 return;
 458         }
 459         TAILQ_FOREACH_FROM_SAFE(q, &tp->t_segq, tqe_q, qtmp) {
 460                 if (SEQ_GT(q->tqe_start, max)) {
 461                         /* Beyond q */
 462                         break;
 463                 }
 464                 /* We have some or all that are overlapping */
 465                 if (SEQ_GEQ(max, (q->tqe_start + q->tqe_len))) {
 466                         /* It consumes it all */
 467                         tp->t_segqmbuflen -= q->tqe_mbuf_cnt;
 468                         m_freem(q->tqe_m);
 469                         TAILQ_REMOVE(&tp->t_segq, q, tqe_q);
 470                         uma_zfree(tcp_reass_zone, q);
 471                         tp->t_segqlen--;
 472                         continue;
 473                 }
 474                 /*
 475                  * Trim the q entry to dovetail to this one
 476                  * and then merge q into ent updating max
 477                  * in the process.
 478                  */
 479                 i = max - q->tqe_start;
 480 #ifdef TCP_REASS_LOGGING
 481                 tcp_log_reassm(tp, q, NULL, 0, i, TCP_R_LOG_TRIM, 2);
 482 #endif
 483                 m_adj(q->tqe_m, i);
 484                 q->tqe_len -= i;
 485                 q->tqe_start += i;
 486                 tcp_reass_merge_into(tp, ent, q);
 487                 max = ent->tqe_start + ent->tqe_len;
 488         }
 489 #ifdef TCP_REASS_COUNTERS
 490         counter_u64_add(merge_fwd, 1);
 491 #endif
 492 }
 493
 494 static int
 495 tcp_reass_overhead_of_chain(struct mbuf *m, struct mbuf **mlast)
 496 {
 497         int len = MSIZE;
 498
 499         if (m->m_flags & M_EXT)
 500                 len += m->m_ext.ext_size;
 501         while (m->m_next != NULL) {
 502                 m = m->m_next;
 503                 len += MSIZE;
 504                 if (m->m_flags & M_EXT)
 505                         len += m->m_ext.ext_size;
 506         }
 507         *mlast = m;
 508         return (len);
 509 }
 510
 511
 512 /*
 513  * NOTE!!! the new tcp-reassembly code *must not* use
 514  * m_adj() with a negative index. That alters the chain
 515  * of mbufs (by possibly chopping trailing mbufs). At
 516  * the front of tcp_reass we count the mbuf overhead
 517  * and setup the tail pointer. If we use m_adj(m, -5)
 518  * we could corrupt the tail pointer. Currently the
 519  * code only uses m_adj(m, postive-num). If this
 520  * changes appropriate changes to update mlast would
 521  * be needed.
 522  */
 523 int
 524 tcp_reass(struct tcpcb *tp, struct tcphdr *th, tcp_seq *seq_start,
 525           int *tlenp, struct mbuf *m)
 526 {
 527         struct tseg_qent *q, *last, *first;
 528         struct tseg_qent *p = NULL;
 529         struct tseg_qent *nq = NULL;
 530         struct tseg_qent *te = NULL;
 531         struct mbuf *mlast = NULL;
 532         struct sockbuf *sb;
 533         struct socket *so = tp->t_inpcb->inp_socket;
 534         char *s = NULL;
 535         int flags, i, lenofoh;
 536
 537         INP_WLOCK_ASSERT(tp->t_inpcb);
 538         /*
 539          * XXX: tcp_reass() is rather inefficient with its data structures
 540          * and should be rewritten (see NetBSD for optimizations).
 541          */
 542
 543         /*
 544          * Call with th==NULL after become established to
 545          * force pre-ESTABLISHED data up to user socket.
 546          */
 547         if (th == NULL)
 548                 goto present;
 549         KASSERT(SEQ_GEQ(th->th_seq, tp->rcv_nxt),
 550                 ("Attempt to add old entry to reassembly queue (th=%p, tp=%p)",
 551                  th, tp));
 552 #ifdef TCP_REASS_LOGGING
 553         tcp_reass_log_new_in(tp, th->th_seq, *tlenp, m, TCP_R_LOG_ADD, NULL);
 554 #endif
 555 #ifdef TCP_REASS_COUNTERS
 556         counter_u64_add(reass_entry, 1);
 557 #endif
 558         /*
 559          * Check for zero length data.
 560          */
 561         if ((*tlenp == 0) && ((th->th_flags & TH_FIN) == 0)) {
 562                 /*
 563                  * A zero length segment does no
 564                  * one any good. We could check
 565                  * the rcv_nxt <-> rcv_wnd but thats
 566                  * already done for us by the caller.
 567                  */
 568 #ifdef TCP_REASS_COUNTERS
 569                 counter_u64_add(tcp_zero_input, 1);
 570 #endif
 571                 m_freem(m);
 572 #ifdef TCP_REASS_LOGGING
 573                 tcp_reass_log_dump(tp);
 574 #endif
 575                 return (0);
 576         }
 577         /*
 578          * Will it fit?
 579          */
 580         lenofoh = tcp_reass_overhead_of_chain(m, &mlast);
 581         sb = &tp->t_inpcb->inp_socket->so_rcv;
 582         if ((th->th_seq != tp->rcv_nxt || !TCPS_HAVEESTABLISHED(tp->t_state)) &&
 583             (sb->sb_mbcnt + tp->t_segqmbuflen + lenofoh) > sb->sb_mbmax) {
 584                 /* No room */
 585                 TCPSTAT_INC(tcps_rcvreassfull);
 586 #ifdef TCP_REASS_COUNTERS
 587                 counter_u64_add(reass_nospace, 1);
 588 #endif
 589 #ifdef TCP_REASS_LOGGING
 590                 tcp_log_reassm(tp, NULL, NULL, th->th_seq, lenofoh, TCP_R_LOG_LIMIT_REACHED, 0);
 591 #endif
 592                 if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL, NULL))) {
 593                         log(LOG_DEBUG, "%s; %s: mbuf count limit reached, "
 594                             "segment dropped\n", s, __func__);
 595                         free(s, M_TCPLOG);
 596                 }
 597                 m_freem(m);
 598                 *tlenp = 0;
 599 #ifdef TCP_REASS_LOGGING
 600                 tcp_reass_log_dump(tp);
 601 #endif
 602                 return (0);
 603         }
 604         /*
 605          * First lets deal with two common cases, the
 606          * segment appends to the back of our collected
 607          * segments. Or the segment is the next in line.
 608          */
 609         last = TAILQ_LAST_FAST(&tp->t_segq, tseg_qent, tqe_q);
 610         if (last != NULL) {
 611                 if ((th->th_flags & TH_FIN) &&
 612                     SEQ_LT((th->th_seq + *tlenp), (last->tqe_start + last->tqe_len))) {
 613                         /*
 614                          * Someone is trying to game us, dump
 615                          * the segment.
 616                          */
 617                         *tlenp = 0;
 618                         m_freem(m);
 619                         return (0);
 620                 }
 621                 if ((SEQ_GEQ(th->th_seq, last->tqe_start)) &&
 622                     (SEQ_GEQ((last->tqe_start + last->tqe_len), th->th_seq))) {
 623                         /* Common case, trailing segment is added */
 624                         /**
 625                          *                                 +--last
 626                          *                                 v
 627                          *  reassembly buffer |---|  |---| |---|
 628                          *  new segment                       |---|
 629                          */
 630 #ifdef TCP_REASS_COUNTERS
 631                         counter_u64_add(reass_path1, 1);
 632 #endif
 633                         if (SEQ_GT((last->tqe_start + last->tqe_len), th->th_seq)) {
 634                                 i = (last->tqe_start + last->tqe_len) - th->th_seq;
 635                                 if (i < *tlenp) {
 636 #ifdef TCP_REASS_LOGGING
 637                                         tcp_log_reassm(tp, last, NULL, 0, i, TCP_R_LOG_TRIM, 3);
 638                                         th->th_seq += i;
 639 #endif
 640                                         m_adj(m, i);
 641                                         *tlenp -= i;
 642                                 } else {
 643                                         /* Complete overlap */
 644                                         TCPSTAT_INC(tcps_rcvduppack);
 645                                         TCPSTAT_ADD(tcps_rcvdupbyte, *tlenp);
 646                                         m_freem(m);
 647                                         *tlenp = last->tqe_len;
 648                                         *seq_start = last->tqe_start;
 649                                         return (0);
 650                                 }
 651                         }
 652                         if (last->tqe_flags & TH_FIN) {
 653                                 /*
 654                                  * We have data after the FIN on the last?
 655                                  */
 656                                 *tlenp = 0;
 657                                 m_freem(m);
 658                                 return(0);
 659                         }
 660                         tcp_reass_append(tp, last, m, th, *tlenp, mlast, lenofoh);
 661                         tp->t_segqmbuflen += lenofoh;
 662                         *seq_start = last->tqe_start;
 663                         *tlenp = last->tqe_len;
 664                         return (0);
 665                 } else if (SEQ_GT(th->th_seq, (last->tqe_start + last->tqe_len))) {
 666                         /*
 667                          * Second common case, we missed
 668                          * another one and have something more
 669                          * for the end.
 670                          */
 671                         /**
 672                          *                                 +--last
 673                          *                                 v
 674                          *  reassembly buffer |---|  |---| |---|
 675                          *  new segment                           |---|
 676                          */
 677                         if (last->tqe_flags & TH_FIN) {
 678                                 /*
 679                                  * We have data after the FIN on the last?
 680                                  */
 681                                 *tlenp = 0;
 682                                 m_freem(m);
 683                                 return(0);
 684                         }
 685 #ifdef TCP_REASS_COUNTERS
 686                         counter_u64_add(reass_path2, 1);
 687 #endif
 688                         p = last;
 689                         goto new_entry;
 690                 }
 691         } else {
 692                 /* First segment (it's NULL). */
 693                 goto new_entry;
 694         }
 695         first = TAILQ_FIRST(&tp->t_segq);
 696         if (SEQ_LT(th->th_seq, first->tqe_start) &&
 697             SEQ_GEQ((th->th_seq + *tlenp),first->tqe_start) &&
 698             SEQ_LT((th->th_seq + *tlenp), (first->tqe_start + first->tqe_len))) {
 699                 /*
 700                  * The head of the queue is prepended by this and
 701                  * it may be the one I want most.
 702                  */
 703                 /**
 704                  *       first-------+
 705                  *                   v
 706                  *  rea:             |---|  |---| |---|
 707                  *  new:         |---|
 708                  * Note the case we do not deal with here is:
 709                  *   rea=     |---|   |---|   |---|
 710                  *   new=  |----|
 711                  * Due to the fact that it could be
 712                  *   new   |--------------------|
 713                  * And we might need to merge forward.
 714                  */
 715 #ifdef INVARIANTS
 716                 struct mbuf *firstmbuf;
 717 #endif
 718
 719 #ifdef TCP_REASS_COUNTERS
 720                 counter_u64_add(reass_path3, 1);
 721 #endif
 722                 if (SEQ_LT(th->th_seq, tp->rcv_nxt)) {
 723                         /*
 724                          * The resend was even before
 725                          * what we have. We need to trim it.
 726                          * Note TSNH (it should be trimmed
 727                          * before the call to tcp_reass()).
 728                          */
 729 #ifdef INVARIANTS
 730                         panic("th->th_seq:%u rcv_nxt:%u tp:%p not pre-trimmed",
 731                               th->th_seq, tp->rcv_nxt, tp);
 732 #else
 733                         i = tp->rcv_nxt - th->th_seq;
 734 #ifdef TCP_REASS_LOGGING
 735                         tcp_log_reassm(tp, first, NULL, 0, i, TCP_R_LOG_TRIM, 4);
 736 #endif
 737                         m_adj(m, i);
 738                         th->th_seq += i;
 739                         *tlenp -= i;
 740 #endif
 741                 }
 742 #ifdef INVARIANTS
 743                 firstmbuf = first->tqe_m;
 744 #endif
 745                 tcp_reass_prepend(tp, first, m, th, *tlenp, mlast, lenofoh);
 746 #ifdef INVARIANTS
 747                 if (firstmbuf == first->tqe_m) {
 748                         panic("First stayed same m:%p foobar:%p first->tqe_m:%p tp:%p first:%p",
 749                               m, firstmbuf, first->tqe_m, tp, first);
 750                 } else if (first->tqe_m != m) {
 751                         panic("First did not change to m:%p foobar:%p first->tqe_m:%p tp:%p first:%p",
 752                               m, firstmbuf, first->tqe_m, tp, first);
 753                 }
 754 #endif
 755                 tp->t_segqmbuflen += lenofoh;
 756                 *seq_start = first->tqe_start;
 757                 *tlenp = first->tqe_len;
 758                 goto present;
 759         } else if (SEQ_LT((th->th_seq + *tlenp), first->tqe_start)) {
 760                 /* New segment is before our earliest segment. */
 761                 /**
 762                  *           first---->+
 763                  *                      v
 764                  *  rea=                |---| ....
 765                  *  new"         |---|
 766                  *
 767                  */
 768                 goto new_entry;
 769         }
 770         /*
 771          * Find a segment which begins after this one does.
 772          */
 773 #ifdef TCP_REASS_COUNTERS
 774         counter_u64_add(reass_fullwalk, 1);
 775 #endif
 776         TAILQ_FOREACH(q, &tp->t_segq, tqe_q) {
 777                 if (SEQ_GT(q->tqe_start, th->th_seq))
 778                         break;
 779         }
 780         p = TAILQ_PREV(q, tsegqe_head, tqe_q);
 781         /**
 782          * Now is this fit just in-between only?
 783          * i.e.:
 784          *      p---+        +----q
 785          *          v        v
 786          *     res= |--|     |--|    |--|
 787          *     nee       |-|
 788          */
 789         if (SEQ_LT((th->th_seq + *tlenp), q->tqe_start) &&
 790             ((p == NULL) || (SEQ_GT(th->th_seq, (p->tqe_start + p->tqe_len))))) {
 791                 /* Yep no overlap */
 792                 goto new_entry;
 793         }
 794         /**
 795          * If we reach here we have some (possibly all) overlap
 796          * such as:
 797          *     res=     |--|     |--|    |--|
 798          *     new=  |----|
 799          * or  new=  |-----------------|
 800          * or  new=      |--------|
 801          * or  new=            |---|
 802          * or  new=            |-----------|
 803          */
 804         if ((p != NULL) &&
 805             (SEQ_LEQ(th->th_seq, (p->tqe_start + p->tqe_len)))) {
 806                 /* conversion to int (in i) handles seq wraparound */
 807
 808 #ifdef TCP_REASS_COUNTERS
 809                 counter_u64_add(reass_path4, 1);
 810 #endif
 811                 i = p->tqe_start + p->tqe_len - th->th_seq;
 812                 if (i >= 0) {
 813                         if (i >= *tlenp) {
 814                                 /**
 815                                  *       prev seg---->+
 816                                  *                    v
 817                                  *  reassembly buffer |---|
 818                                  *  new segment        |-|
 819                                  */
 820                                 TCPSTAT_INC(tcps_rcvduppack);
 821                                 TCPSTAT_ADD(tcps_rcvdupbyte, *tlenp);
 822                                 *tlenp = p->tqe_len;
 823                                 *seq_start = p->tqe_start;
 824                                 m_freem(m);
 825                                 /*
 826                                  * Try to present any queued data
 827                                  * at the left window edge to the user.
 828                                  * This is needed after the 3-WHS
 829                                  * completes. Note this probably
 830                                  * will not work and we will return.
 831                                  */
 832                                 return (0);
 833                         }
 834                         if (i > 0) {
 835                                 /**
 836                                  *       prev seg---->+
 837                                  *                    v
 838                                  *  reassembly buffer |---|
 839                                  *  new segment         |-----|
 840                                  */
 841 #ifdef TCP_REASS_COUNTERS
 842                                 counter_u64_add(reass_path5, 1);
 843 #endif
 844 #ifdef TCP_REASS_LOGGING
 845                                 tcp_log_reassm(tp, p, NULL, 0, i, TCP_R_LOG_TRIM, 5);
 846 #endif
 847                                 m_adj(m, i);
 848                                 *tlenp -= i;
 849                                 th->th_seq += i;
 850                         }
 851                 }
 852                 if (th->th_seq == (p->tqe_start + p->tqe_len)) {
 853                         /*
 854                          * If dovetails in with this one
 855                          * append it.
 856                          */
 857                         /**
 858                          *       prev seg---->+
 859                          *                    v
 860                          *  reassembly buffer |--|     |---|
 861                          *  new segment          |--|
 862                          * (note: it was trimmed above if it overlapped)
 863                          */
 864                         tcp_reass_append(tp, p, m, th, *tlenp, mlast, lenofoh);
 865                         tp->t_segqmbuflen += lenofoh;
 866                 } else {
 867 #ifdef INVARIANTS
 868                         panic("Impossible cut th_seq:%u p->seq:%u(%d) p:%p tp:%p",
 869                               th->th_seq, p->tqe_start, p->tqe_len,
 870                               p, tp);
 871 #endif
 872                         *tlenp = 0;
 873                         m_freem(m);
 874                         return (0);
 875                 }
 876                 q = p;
 877         } else {
 878                 /*
 879                  * The new data runs over the
 880                  * top of previously sack'd data (in q).
 881                  * It may be partially overlapping, or
 882                  * it may overlap the entire segment.
 883                  */
 884 #ifdef TCP_REASS_COUNTERS
 885                 counter_u64_add(reass_path6, 1);
 886 #endif
 887                 if (SEQ_GEQ((th->th_seq + *tlenp), (q->tqe_start + q->tqe_len))) {
 888                         /* It consumes it all */
 889                         /**
 890                          *             next seg---->+
 891                          *                          v
 892                          *  reassembly buffer |--|     |---|
 893                          *  new segment              |----------|
 894                          */
 895 #ifdef TCP_REASS_COUNTERS
 896                         counter_u64_add(reass_path7, 1);
 897 #endif
 898                         tcp_reass_replace(tp, q, m, th->th_seq, *tlenp, mlast, lenofoh, th->th_flags);
 899                 } else {
 900                         /*
 901                          * We just need to prepend the data
 902                          * to this. It does not overrun
 903                          * the end.
 904                          */
 905                         /**
 906                          *                next seg---->+
 907                          *                             v
 908                          *  reassembly buffer |--|     |---|
 909                          *  new segment                   |----------|
 910                          */
 911                         tcp_reass_prepend(tp, q, m, th, *tlenp, mlast, lenofoh);
 912                         tp->t_segqmbuflen += lenofoh;
 913                 }
 914         }
 915         /* Now does it go further than that? */
 916         tcp_reass_merge_forward(tp, q);
 917         *seq_start = q->tqe_start;
 918         *tlenp = q->tqe_len;
 919         goto present;
 920
 921         /*
 922          * When we reach here we can't combine it
 923          * with any existing segment.
 924          *
 925          * Limit the number of segments that can be queued to reduce the
 926          * potential for mbuf exhaustion. For best performance, we want to be
 927          * able to queue a full window's worth of segments. The size of the
 928          * socket receive buffer determines our advertised window and grows
 929          * automatically when socket buffer autotuning is enabled. Use it as the
 930          * basis for our queue limit.
 931          *
 932          * However, allow the user to specify a ceiling for the number of
 933          * segments in each queue.
 934          *
 935          * Always let the missing segment through which caused this queue.
 936          * NB: Access to the socket buffer is left intentionally unlocked as we
 937          * can tolerate stale information here.
 938          *
 939          * XXXLAS: Using sbspace(so->so_rcv) instead of so->so_rcv.sb_hiwat
 940          * should work but causes packets to be dropped when they shouldn't.
 941          * Investigate why and re-evaluate the below limit after the behaviour
 942          * is understood.
 943          */
 944 new_entry:
 945         if (th->th_seq == tp->rcv_nxt && TCPS_HAVEESTABLISHED(tp->t_state)) {
 946                 tp->rcv_nxt += *tlenp;
 947                 flags = th->th_flags & TH_FIN;
 948                 TCPSTAT_INC(tcps_rcvoopack);
 949                 TCPSTAT_ADD(tcps_rcvoobyte, *tlenp);
 950                 SOCKBUF_LOCK(&so->so_rcv);
 951                 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 952                         m_freem(m);
 953                 } else {
 954                         sbappendstream_locked(&so->so_rcv, m, 0);
 955                 }
 956                 sorwakeup_locked(so);
 957                 return (flags);
 958         }
 959         if (tcp_new_limits) {
 960                 if ((tp->t_segqlen > tcp_reass_queue_guard) &&
 961                     (*tlenp < MSIZE)) {
 962                         /*
 963                          * This is really a lie, we are not full but
 964                          * are getting a segment that is above
 965                          * guard threshold. If it is and its below
 966                          * a mbuf size (256) we drop it if it
 967                          * can't fill in some place.
 968                          */
 969                         TCPSTAT_INC(tcps_rcvreassfull);
 970                         *tlenp = 0;
 971                         if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL, NULL))) {
 972                                 log(LOG_DEBUG, "%s; %s: queue limit reached, "
 973                                     "segment dropped\n", s, __func__);
 974                                 free(s, M_TCPLOG);
 975                         }
 976                         m_freem(m);
 977 #ifdef TCP_REASS_LOGGING
 978                         tcp_reass_log_dump(tp);
 979 #endif
 980                         return (0);
 981                 }
 982         } else {
 983                 if (tp->t_segqlen >= min((so->so_rcv.sb_hiwat / tp->t_maxseg) + 1,
 984                                          tcp_reass_maxqueuelen)) {
 985                         TCPSTAT_INC(tcps_rcvreassfull);
 986                         *tlenp = 0;
 987                         if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL, NULL))) {
 988                                 log(LOG_DEBUG, "%s; %s: queue limit reached, "
 989                                     "segment dropped\n", s, __func__);
 990                                 free(s, M_TCPLOG);
 991                         }
 992                         m_freem(m);
 993 #ifdef TCP_REASS_LOGGING
 994                         tcp_reass_log_dump(tp);
 995 #endif
 996                         return (0);
 997                 }
 998         }
 999         /*
1000          * Allocate a new queue entry. If we can't, or hit the zone limit
1001          * just drop the pkt.
1002          */
1003         te = uma_zalloc(tcp_reass_zone, M_NOWAIT);
1004         if (te == NULL) {
1005                 TCPSTAT_INC(tcps_rcvmemdrop);
1006                 m_freem(m);
1007                 *tlenp = 0;
1008                 if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL,
1009                                        NULL))) {
1010                         log(LOG_DEBUG, "%s; %s: global zone limit "
1011                             "reached, segment dropped\n", s, __func__);
1012                         free(s, M_TCPLOG);
1013                 }
1014                 return (0);
1015         }
1016         tp->t_segqlen++;
1017         tp->t_rcvoopack++;
1018         TCPSTAT_INC(tcps_rcvoopack);
1019         TCPSTAT_ADD(tcps_rcvoobyte, *tlenp);
1020         /* Insert the new segment queue entry into place. */
1021         te->tqe_m = m;
1022         te->tqe_flags = th->th_flags;
1023         te->tqe_len = *tlenp;
1024         te->tqe_start = th->th_seq;
1025         te->tqe_last = mlast;
1026         te->tqe_mbuf_cnt = lenofoh;
1027         tp->t_segqmbuflen += te->tqe_mbuf_cnt;
1028         if (p == NULL) {
1029                 TAILQ_INSERT_HEAD(&tp->t_segq, te, tqe_q);
1030         } else {
1031                 TAILQ_INSERT_AFTER(&tp->t_segq, p, te, tqe_q);
1032         }
1033 #ifdef TCP_REASS_LOGGING
1034         tcp_reass_log_new_in(tp, th->th_seq, *tlenp, m, TCP_R_LOG_NEW_ENTRY, te);
1035 #endif
1036 present:
1037         /*
1038          * Present data to user, advancing rcv_nxt through
1039          * completed sequence space.
1040          */
1041         if (!TCPS_HAVEESTABLISHED(tp->t_state))
1042                 return (0);
1043         q = TAILQ_FIRST(&tp->t_segq);
1044         KASSERT(q == NULL || SEQ_GEQ(q->tqe_start, tp->rcv_nxt),
1045                 ("Reassembly queue for %p has stale entry at head", tp));
1046         if (!q || q->tqe_start != tp->rcv_nxt) {
1047 #ifdef TCP_REASS_LOGGING
1048                 tcp_reass_log_dump(tp);
1049 #endif
1050                 return (0);
1051         }
1052         SOCKBUF_LOCK(&so->so_rcv);
1053         do {
1054                 tp->rcv_nxt += q->tqe_len;
1055                 flags = q->tqe_flags & TH_FIN;
1056                 nq = TAILQ_NEXT(q, tqe_q);
1057                 TAILQ_REMOVE(&tp->t_segq, q, tqe_q);
1058                 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
1059                         m_freem(q->tqe_m);
1060                 } else {
1061 #ifdef TCP_REASS_LOGGING
1062                         tcp_reass_log_new_in(tp, q->tqe_start, q->tqe_len, q->tqe_m, TCP_R_LOG_READ, q);
1063                         tcp_log_reassm(tp, q, NULL, th->th_seq, *tlenp, TCP_R_LOG_READ, 1);
1064 #endif
1065                         sbappendstream_locked(&so->so_rcv, q->tqe_m, 0);
1066                 }
1067 #ifdef TCP_REASS_LOGGING
1068                 tcp_log_reassm(tp, q, NULL, th->th_seq, *tlenp, TCP_R_LOG_READ, 2);
1069 #endif
1070                 KASSERT(tp->t_segqmbuflen >= q->tqe_mbuf_cnt,
1071                         ("tp:%p seg queue goes negative", tp));
1072                 tp->t_segqmbuflen -= q->tqe_mbuf_cnt;
1073                 uma_zfree(tcp_reass_zone, q);
1074                 tp->t_segqlen--;
1075                 q = nq;
1076         } while (q && q->tqe_start == tp->rcv_nxt);
1077         if (TAILQ_EMPTY(&tp->t_segq) &&
1078             (tp->t_segqmbuflen != 0)) {
1079 #ifdef INVARIANTS
1080                 panic("tp:%p segq:%p len:%d queue empty",
1081                       tp, &tp->t_segq, tp->t_segqmbuflen);
1082 #else
1083 #ifdef TCP_REASS_LOGGING
1084                 tcp_log_reassm(tp, NULL, NULL, th->th_seq, *tlenp, TCP_R_LOG_ZERO, 0);
1085 #endif
1086                 tp->t_segqmbuflen = 0;
1087 #endif
1088         }
1089 #ifdef TCP_REASS_LOGGING
1090         tcp_reass_log_dump(tp);
1091 #endif
1092         sorwakeup_locked(so);
1093         return (flags);
1094 }