sys/netinet/tcp_reass.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1994, 1995
   5  *      The Regents of the University of California.  All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. Neither the name of the University nor the names of its contributors
  16  *    may be used to endorse or promote products derived from this software
  17  *    without specific prior written permission.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31  *      @(#)tcp_input.c 8.12 (Berkeley) 5/24/95
  32  */
  33
  34 #include <sys/cdefs.h>
  35 __FBSDID("$FreeBSD$");
  36
  37 #include "opt_inet.h"
  38 #include "opt_inet6.h"
  39 #include "opt_tcpdebug.h"
  40
  41 #include <sys/param.h>
  42 #include <sys/kernel.h>
  43 #include <sys/eventhandler.h>
  44 #include <sys/malloc.h>
  45 #include <sys/mbuf.h>
  46 #include <sys/socket.h>
  47 #include <sys/socketvar.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/syslog.h>
  50 #include <sys/systm.h>
  51
  52 #include <vm/uma.h>
  53
  54 #include <net/if.h>
  55 #include <net/if_var.h>
  56 #include <net/route.h>
  57 #include <net/vnet.h>
  58
  59 #include <netinet/in.h>
  60 #include <netinet/in_pcb.h>
  61 #include <netinet/in_systm.h>
  62 #include <netinet/in_var.h>
  63 #include <netinet/ip.h>
  64 #include <netinet/ip_var.h>
  65 #include <netinet/ip_options.h>
  66 #include <netinet/ip6.h>
  67 #include <netinet6/in6_pcb.h>
  68 #include <netinet6/ip6_var.h>
  69 #include <netinet6/nd6.h>
  70 #include <netinet/tcp.h>
  71 #include <netinet/tcp_fsm.h>
  72 #include <netinet/tcp_seq.h>
  73 #include <netinet/tcp_timer.h>
  74 #include <netinet/tcp_var.h>
  75 #include <netinet/tcp_log_buf.h>
  76 #include <netinet/tcp_hpts.h>
  77 #include <netinet6/tcp6_var.h>
  78 #include <netinet/tcpip.h>
  79 #ifdef TCPDEBUG
  80 #include <netinet/tcp_debug.h>
  81 #endif /* TCPDEBUG */
  82
  83 #define TCP_R_LOG_ADD           1
  84 #define TCP_R_LOG_LIMIT_REACHED 2
  85 #define TCP_R_LOG_APPEND        3
  86 #define TCP_R_LOG_PREPEND       4
  87 #define TCP_R_LOG_REPLACE       5
  88 #define TCP_R_LOG_MERGE_INTO    6
  89 #define TCP_R_LOG_NEW_ENTRY     7
  90 #define TCP_R_LOG_READ          8
  91 #define TCP_R_LOG_ZERO          9
  92 #define TCP_R_LOG_DUMP          10
  93 #define TCP_R_LOG_TRIM          11
  94
  95 /* For debugging we want counters and BB logging */
  96 /* #define TCP_REASS_COUNTERS 1 */
  97 /* #define TCP_REASS_LOGGING 1 */
  98
  99 static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, reass, CTLFLAG_RW, 0,
 100     "TCP Segment Reassembly Queue");
 101
 102 static SYSCTL_NODE(_net_inet_tcp_reass, OID_AUTO, stats, CTLFLAG_RW, 0,
 103     "TCP Segment Reassembly stats");
 104
 105
 106 static int tcp_reass_maxseg = 0;
 107 SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, maxsegments, CTLFLAG_RDTUN,
 108     &tcp_reass_maxseg, 0,
 109     "Global maximum number of TCP Segments in Reassembly Queue");
 110
 111 static uma_zone_t tcp_reass_zone;
 112 SYSCTL_UMA_CUR(_net_inet_tcp_reass, OID_AUTO, cursegments, 0,
 113     &tcp_reass_zone,
 114     "Global number of TCP Segments currently in Reassembly Queue");
 115
 116 static u_int tcp_reass_maxqueuelen = 100;
 117 SYSCTL_UINT(_net_inet_tcp_reass, OID_AUTO, maxqueuelen, CTLFLAG_RWTUN,
 118     &tcp_reass_maxqueuelen, 0,
 119     "Maximum number of TCP Segments per Reassembly Queue");
 120
 121 static int tcp_new_limits = 0;
 122 SYSCTL_INT(_net_inet_tcp_reass, OID_AUTO, new_limit, CTLFLAG_RWTUN,
 123     &tcp_new_limits, 0,
 124     "Do we use the new limit method we are discussing?");
 125
 126 static u_int tcp_reass_queue_guard = 16;
 127 SYSCTL_UINT(_net_inet_tcp_reass, OID_AUTO, queueguard, CTLFLAG_RWTUN,
 128     &tcp_reass_queue_guard, 16,
 129     "Number of TCP Segments in Reassembly Queue where we flip over to guard mode");
 130
 131 #ifdef TCP_REASS_COUNTERS
 132
 133 counter_u64_t reass_entry;
 134 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, entry, CTLFLAG_RD,
 135     &reass_entry, "A segment entered reassembly ");
 136
 137 counter_u64_t reass_path1;
 138 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path1, CTLFLAG_RD,
 139     &reass_path1, "Took path 1");
 140
 141 counter_u64_t reass_path2;
 142 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path2, CTLFLAG_RD,
 143     &reass_path2, "Took path 2");
 144
 145 counter_u64_t reass_path3;
 146 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path3, CTLFLAG_RD,
 147     &reass_path3, "Took path 3");
 148
 149 counter_u64_t reass_path4;
 150 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path4, CTLFLAG_RD,
 151     &reass_path4, "Took path 4");
 152
 153 counter_u64_t reass_path5;
 154 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path5, CTLFLAG_RD,
 155     &reass_path5, "Took path 5");
 156
 157 counter_u64_t reass_path6;
 158 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path6, CTLFLAG_RD,
 159     &reass_path6, "Took path 6");
 160
 161 counter_u64_t reass_path7;
 162 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, path7, CTLFLAG_RD,
 163     &reass_path7, "Took path 7");
 164
 165 counter_u64_t reass_fullwalk;
 166 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, fullwalk, CTLFLAG_RD,
 167     &reass_fullwalk, "Took a full walk ");
 168
 169 counter_u64_t reass_nospace;
 170 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, nospace, CTLFLAG_RD,
 171     &reass_nospace, "Had no mbuf capacity ");
 172
 173 counter_u64_t merge_fwd;
 174 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, merge_fwd, CTLFLAG_RD,
 175     &merge_fwd, "Ran merge fwd");
 176
 177 counter_u64_t merge_into;
 178 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, merge_into, CTLFLAG_RD,
 179     &merge_into, "Ran merge into");
 180
 181 counter_u64_t tcp_zero_input;
 182 SYSCTL_COUNTER_U64(_net_inet_tcp_reass_stats, OID_AUTO, zero_input, CTLFLAG_RD,
 183     &tcp_zero_input, "The reassembly buffer saw a zero len segment etc");
 184
 185 #endif
 186
 187 /* Initialize TCP reassembly queue */
 188 static void
 189 tcp_reass_zone_change(void *tag)
 190 {
 191
 192         /* Set the zone limit and read back the effective value. */
 193         tcp_reass_maxseg = nmbclusters / 16;
 194         tcp_reass_maxseg = uma_zone_set_max(tcp_reass_zone,
 195             tcp_reass_maxseg);
 196 }
 197
 198 #ifdef TCP_REASS_LOGGING
 199
 200 static void
 201 tcp_log_reassm(struct tcpcb *tp, struct tseg_qent *q, struct tseg_qent *p,
 202     tcp_seq seq, int len, uint8_t action, int instance)
 203 {
 204         uint32_t cts;
 205         struct timeval tv;
 206
 207         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
 208                 union tcp_log_stackspecific log;
 209
 210                 memset(&log, 0, sizeof(log));
 211                 cts = tcp_get_usecs(&tv);
 212                 log.u_bbr.flex1 = seq;
 213                 log.u_bbr.cur_del_rate = (uint64_t)q;
 214                 log.u_bbr.delRate = (uint64_t)p;
 215                 if (q != NULL) {
 216                         log.u_bbr.flex2 = q->tqe_start;
 217                         log.u_bbr.flex3 = q->tqe_len;
 218                         log.u_bbr.flex4 = q->tqe_mbuf_cnt;
 219                         log.u_bbr.hptsi_gain = q->tqe_flags;
 220                 }
 221                 if (p != NULL)  {
 222                         log.u_bbr.flex5 = p->tqe_start;
 223                         log.u_bbr.pkts_out = p->tqe_len;
 224                         log.u_bbr.epoch = p->tqe_mbuf_cnt;
 225                         log.u_bbr.cwnd_gain = p->tqe_flags;
 226                 }
 227                 log.u_bbr.flex6 = tp->t_segqmbuflen;
 228                 log.u_bbr.flex7 = instance;
 229                 log.u_bbr.flex8 = action;
 230                 log.u_bbr.timeStamp = cts;
 231                 TCP_LOG_EVENTP(tp, NULL,
 232                     &tp->t_inpcb->inp_socket->so_rcv,
 233                     &tp->t_inpcb->inp_socket->so_snd,
 234                     TCP_LOG_REASS, 0,
 235                     len, &log, false, &tv);
 236         }
 237 }
 238
 239 static void
 240 tcp_reass_log_dump(struct tcpcb *tp)
 241 {
 242         struct tseg_qent *q;
 243
 244         if (tp->t_logstate != TCP_LOG_STATE_OFF) {
 245                 TAILQ_FOREACH(q, &tp->t_segq, tqe_q) {
 246                         tcp_log_reassm(tp, q, NULL, q->tqe_start, q->tqe_len, TCP_R_LOG_DUMP, 0);
 247                 }
 248         };
 249 }
 250
 251 static void
 252 tcp_reass_log_new_in(struct tcpcb *tp, tcp_seq seq, int len, struct mbuf *m,
 253     int logval, struct tseg_qent *q)
 254 {
 255         int cnt;
 256         struct mbuf *t;
 257
 258         cnt = 0;
 259         t = m;
 260         while (t) {
 261                 cnt += t->m_len;
 262                 t = t->m_next;
 263         }
 264         tcp_log_reassm(tp, q, NULL, seq, len, logval, cnt);
 265 }
 266
 267 #endif
 268
 269 void
 270 tcp_reass_global_init(void)
 271 {
 272
 273         tcp_reass_maxseg = nmbclusters / 16;
 274         TUNABLE_INT_FETCH("net.inet.tcp.reass.maxsegments",
 275             &tcp_reass_maxseg);
 276         tcp_reass_zone = uma_zcreate("tcpreass", sizeof (struct tseg_qent),
 277             NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 278         /* Set the zone limit and read back the effective value. */
 279         tcp_reass_maxseg = uma_zone_set_max(tcp_reass_zone,
 280             tcp_reass_maxseg);
 281 #ifdef TCP_REASS_COUNTERS
 282         reass_path1 = counter_u64_alloc(M_WAITOK);
 283         reass_path2 = counter_u64_alloc(M_WAITOK);
 284         reass_path3 = counter_u64_alloc(M_WAITOK);
 285         reass_path4 = counter_u64_alloc(M_WAITOK);
 286         reass_path5 = counter_u64_alloc(M_WAITOK);
 287         reass_path6 = counter_u64_alloc(M_WAITOK);
 288         reass_path7 = counter_u64_alloc(M_WAITOK);
 289         reass_fullwalk = counter_u64_alloc(M_WAITOK);
 290         reass_nospace = counter_u64_alloc(M_WAITOK);
 291         reass_entry = counter_u64_alloc(M_WAITOK);
 292         merge_fwd = counter_u64_alloc(M_WAITOK);
 293         merge_into = counter_u64_alloc(M_WAITOK);
 294         tcp_zero_input = counter_u64_alloc(M_WAITOK);
 295 #endif
 296         EVENTHANDLER_REGISTER(nmbclusters_change,
 297             tcp_reass_zone_change, NULL, EVENTHANDLER_PRI_ANY);
 298
 299 }
 300
 301 void
 302 tcp_reass_flush(struct tcpcb *tp)
 303 {
 304         struct tseg_qent *qe;
 305
 306         INP_WLOCK_ASSERT(tp->t_inpcb);
 307
 308         while ((qe = TAILQ_FIRST(&tp->t_segq)) != NULL) {
 309                 TAILQ_REMOVE(&tp->t_segq, qe, tqe_q);
 310                 m_freem(qe->tqe_m);
 311                 uma_zfree(tcp_reass_zone, qe);
 312                 tp->t_segqlen--;
 313         }
 314         tp->t_segqmbuflen = 0;
 315         KASSERT((tp->t_segqlen == 0),
 316             ("TCP reass queue %p segment count is %d instead of 0 after flush.",
 317             tp, tp->t_segqlen));
 318 }
 319
 320 static void
 321 tcp_reass_append(struct tcpcb *tp, struct tseg_qent *last,
 322     struct mbuf *m, struct tcphdr *th, int tlen,
 323     struct mbuf *mlast, int lenofoh)
 324 {
 325
 326 #ifdef TCP_REASS_LOGGING
 327         tcp_log_reassm(tp, last, NULL, th->th_seq, tlen, TCP_R_LOG_APPEND, 0);
 328 #endif
 329         last->tqe_len += tlen;
 330         last->tqe_m->m_pkthdr.len += tlen;
 331         /* Preserve the FIN bit if its there */
 332         last->tqe_flags |= (th->th_flags & TH_FIN);
 333         last->tqe_last->m_next = m;
 334         last->tqe_last = mlast;
 335         last->tqe_mbuf_cnt += lenofoh;
 336         tp->t_rcvoopack++;
 337         TCPSTAT_INC(tcps_rcvoopack);
 338         TCPSTAT_ADD(tcps_rcvoobyte, tlen);
 339 #ifdef TCP_REASS_LOGGING
 340         tcp_reass_log_new_in(tp, last->tqe_start, lenofoh, last->tqe_m,
 341                              TCP_R_LOG_APPEND,
 342                              last);
 343 #endif
 344 }
 345
 346 static void
 347 tcp_reass_prepend(struct tcpcb *tp, struct tseg_qent *first, struct mbuf *m, struct tcphdr *th,
 348                   int tlen, struct mbuf *mlast, int lenofoh)
 349 {
 350         int i;
 351
 352 #ifdef TCP_REASS_LOGGING
 353         tcp_log_reassm(tp, first, NULL, th->th_seq, tlen, TCP_R_LOG_PREPEND, 0);
 354 #endif
 355         if (SEQ_GT((th->th_seq + tlen), first->tqe_start)) {
 356                 /* The new data overlaps into the old */
 357                 i = (th->th_seq + tlen) - first->tqe_start;
 358 #ifdef TCP_REASS_LOGGING
 359                 tcp_log_reassm(tp, first, NULL, 0, i, TCP_R_LOG_TRIM, 1);
 360 #endif
 361                 m_adj(first->tqe_m, i);
 362                 first->tqe_len -= i;
 363                 first->tqe_start += i;
 364         }
 365         /* Ok now setup our chain to point to the old first */
 366         mlast->m_next = first->tqe_m;
 367         first->tqe_m = m;
 368         first->tqe_len += tlen;
 369         first->tqe_start = th->th_seq;
 370         first->tqe_m->m_pkthdr.len = first->tqe_len;
 371         first->tqe_mbuf_cnt += lenofoh;
 372         tp->t_rcvoopack++;
 373         TCPSTAT_INC(tcps_rcvoopack);
 374         TCPSTAT_ADD(tcps_rcvoobyte, tlen);
 375 #ifdef TCP_REASS_LOGGING
 376         tcp_reass_log_new_in(tp, first->tqe_start, lenofoh, first->tqe_m,
 377                              TCP_R_LOG_PREPEND,
 378                              first);
 379 #endif
 380 }
 381
 382 static void
 383 tcp_reass_replace(struct tcpcb *tp, struct tseg_qent *q, struct mbuf *m,
 384     tcp_seq seq, int len, struct mbuf *mlast, int mbufoh, uint8_t flags)
 385 {
 386         /*
 387          * Free the data in q, and replace
 388          * it with the new segment.
 389          */
 390         int len_dif;
 391
 392 #ifdef TCP_REASS_LOGGING
 393         tcp_log_reassm(tp, q, NULL, seq, len, TCP_R_LOG_REPLACE, 0);
 394 #endif
 395         m_freem(q->tqe_m);
 396         KASSERT(tp->t_segqmbuflen >= q->tqe_mbuf_cnt,
 397                 ("Tp:%p seg queue goes negative", tp));
 398         tp->t_segqmbuflen -= q->tqe_mbuf_cnt;
 399         q->tqe_mbuf_cnt = mbufoh;
 400         q->tqe_m = m;
 401         q->tqe_last = mlast;
 402         q->tqe_start = seq;
 403         if (len > q->tqe_len)
 404                 len_dif = len - q->tqe_len;
 405         else
 406                 len_dif = 0;
 407         tp->t_rcvoopack++;
 408         TCPSTAT_INC(tcps_rcvoopack);
 409         TCPSTAT_ADD(tcps_rcvoobyte, len_dif);
 410         q->tqe_len = len;
 411         q->tqe_flags = (flags & TH_FIN);
 412         q->tqe_m->m_pkthdr.len = q->tqe_len;
 413         tp->t_segqmbuflen += mbufoh;
 414
 415 }
 416
 417 static void
 418 tcp_reass_merge_into(struct tcpcb *tp, struct tseg_qent *ent,
 419     struct tseg_qent *q)
 420 {
 421         /*
 422          * Merge q into ent and free q from the list.
 423          */
 424 #ifdef TCP_REASS_LOGGING
 425         tcp_log_reassm(tp, q, ent, 0, 0, TCP_R_LOG_MERGE_INTO, 0);
 426 #endif
 427 #ifdef TCP_REASS_COUNTERS
 428         counter_u64_add(merge_into, 1);
 429 #endif
 430         ent->tqe_last->m_next = q->tqe_m;
 431         ent->tqe_last = q->tqe_last;
 432         ent->tqe_len += q->tqe_len;
 433         ent->tqe_mbuf_cnt += q->tqe_mbuf_cnt;
 434         ent->tqe_m->m_pkthdr.len += q->tqe_len;
 435         ent->tqe_flags |= (q->tqe_flags & TH_FIN);
 436         TAILQ_REMOVE(&tp->t_segq, q, tqe_q);
 437         uma_zfree(tcp_reass_zone, q);
 438         tp->t_segqlen--;
 439
 440 }
 441
 442 static void
 443 tcp_reass_merge_forward(struct tcpcb *tp, struct tseg_qent *ent)
 444 {
 445         struct tseg_qent *q, *qtmp;
 446         int i;
 447         tcp_seq max;
 448         /*
 449          * Given an entry merge forward anyplace
 450          * that ent overlaps forward.
 451          */
 452
 453         max = ent->tqe_start + ent->tqe_len;
 454         q = TAILQ_NEXT(ent, tqe_q);
 455         if (q == NULL) {
 456                 /* Nothing left */
 457                 return;
 458         }
 459         TAILQ_FOREACH_FROM_SAFE(q, &tp->t_segq, tqe_q, qtmp) {
 460                 if (SEQ_GT(q->tqe_start, max)) {
 461                         /* Beyond q */
 462                         break;
 463                 }
 464                 /* We have some or all that are overlapping */
 465                 if (SEQ_GEQ(max, (q->tqe_start + q->tqe_len))) {
 466                         /* It consumes it all */
 467                         tp->t_segqmbuflen -= q->tqe_mbuf_cnt;
 468                         m_freem(q->tqe_m);
 469                         TAILQ_REMOVE(&tp->t_segq, q, tqe_q);
 470                         uma_zfree(tcp_reass_zone, q);
 471                         tp->t_segqlen--;
 472                         continue;
 473                 }
 474                 /*
 475                  * Trim the q entry to dovetail to this one
 476                  * and then merge q into ent updating max
 477                  * in the process.
 478                  */
 479                 i = max - q->tqe_start;
 480 #ifdef TCP_REASS_LOGGING
 481                 tcp_log_reassm(tp, q, NULL, 0, i, TCP_R_LOG_TRIM, 2);
 482 #endif
 483                 m_adj(q->tqe_m, i);
 484                 q->tqe_len -= i;
 485                 q->tqe_start += i;
 486                 tcp_reass_merge_into(tp, ent, q);
 487                 max = ent->tqe_start + ent->tqe_len;
 488         }
 489 #ifdef TCP_REASS_COUNTERS
 490         counter_u64_add(merge_fwd, 1);
 491 #endif
 492 }
 493
 494 static int
 495 tcp_reass_overhead_of_chain(struct mbuf *m, struct mbuf **mlast)
 496 {
 497         int len = MSIZE;
 498
 499         if (m->m_flags & M_EXT)
 500                 len += m->m_ext.ext_size;
 501         while (m->m_next != NULL) {
 502                 m = m->m_next;
 503                 len += MSIZE;
 504                 if (m->m_flags & M_EXT)
 505                         len += m->m_ext.ext_size;
 506         }
 507         *mlast = m;
 508         return (len);
 509 }
 510
 511
 512 /*
 513  * NOTE!!! the new tcp-reassembly code *must not* use
 514  * m_adj() with a negative index. That alters the chain
 515  * of mbufs (by possibly chopping trailing mbufs). At
 516  * the front of tcp_reass we count the mbuf overhead
 517  * and setup the tail pointer. If we use m_adj(m, -5)
 518  * we could corrupt the tail pointer. Currently the
 519  * code only uses m_adj(m, postive-num). If this
 520  * changes appropriate changes to update mlast would
 521  * be needed.
 522  */
 523 int
 524 tcp_reass(struct tcpcb *tp, struct tcphdr *th, tcp_seq *seq_start,
 525           int *tlenp, struct mbuf *m)
 526 {
 527         struct tseg_qent *q, *last, *first;
 528         struct tseg_qent *p = NULL;
 529         struct tseg_qent *nq = NULL;
 530         struct tseg_qent *te = NULL;
 531         struct mbuf *mlast = NULL;
 532         struct sockbuf *sb;
 533         struct socket *so = tp->t_inpcb->inp_socket;
 534         char *s = NULL;
 535         int flags, i, lenofoh;
 536
 537         INP_WLOCK_ASSERT(tp->t_inpcb);
 538         /*
 539          * XXX: tcp_reass() is rather inefficient with its data structures
 540          * and should be rewritten (see NetBSD for optimizations).
 541          */
 542
 543         /*
 544          * Call with th==NULL after become established to
 545          * force pre-ESTABLISHED data up to user socket.
 546          */
 547         if (th == NULL)
 548                 goto present;
 549         KASSERT(SEQ_GEQ(th->th_seq, tp->rcv_nxt),
 550                 ("Attempt to add old entry to reassembly queue (th=%p, tp=%p)",
 551                  th, tp));
 552 #ifdef TCP_REASS_LOGGING
 553         tcp_reass_log_new_in(tp, th->th_seq, *tlenp, m, TCP_R_LOG_ADD, NULL);
 554 #endif
 555 #ifdef TCP_REASS_COUNTERS
 556         counter_u64_add(reass_entry, 1);
 557 #endif
 558         /*
 559          * Check for zero length data.
 560          */
 561         if ((*tlenp == 0) && ((th->th_flags & TH_FIN) == 0)) {
 562                 /*
 563                  * A zero length segment does no
 564                  * one any good. We could check
 565                  * the rcv_nxt <-> rcv_wnd but thats
 566                  * already done for us by the caller.
 567                  */
 568 #ifdef TCP_REASS_COUNTERS
 569                 counter_u64_add(tcp_zero_input, 1);
 570 #endif
 571                 m_freem(m);
 572 #ifdef TCP_REASS_LOGGING
 573                 tcp_reass_log_dump(tp);
 574 #endif
 575                 return (0);
 576         }
 577         /*
 578          * Will it fit?
 579          */
 580         lenofoh = tcp_reass_overhead_of_chain(m, &mlast);
 581         sb = &tp->t_inpcb->inp_socket->so_rcv;
 582         if ((sb->sb_mbcnt + tp->t_segqmbuflen + lenofoh) > sb->sb_mbmax) {
 583                 /* No room */
 584                 TCPSTAT_INC(tcps_rcvreassfull);
 585 #ifdef TCP_REASS_COUNTERS
 586                 counter_u64_add(reass_nospace, 1);
 587 #endif
 588 #ifdef TCP_REASS_LOGGING
 589                 tcp_log_reassm(tp, NULL, NULL, th->th_seq, lenofoh, TCP_R_LOG_LIMIT_REACHED, 0);
 590 #endif
 591                 m_freem(m);
 592                 *tlenp = 0;
 593 #ifdef TCP_REASS_LOGGING
 594                 tcp_reass_log_dump(tp);
 595 #endif
 596                 return (0);
 597         }
 598         /*
 599          * First lets deal with two common cases, the
 600          * segment appends to the back of our collected
 601          * segments. Or the segment is the next in line.
 602          */
 603         last = TAILQ_LAST_FAST(&tp->t_segq, tseg_qent, tqe_q);
 604         if (last != NULL) {
 605                 if ((th->th_flags & TH_FIN) &&
 606                     SEQ_LT((th->th_seq + *tlenp), (last->tqe_start + last->tqe_len))) {
 607                         /*
 608                          * Someone is trying to game us, dump
 609                          * the segment.
 610                          */
 611                         *tlenp = 0;
 612                         m_freem(m);
 613                         return (0);
 614                 }
 615                 if ((SEQ_GEQ(th->th_seq, last->tqe_start)) &&
 616                     (SEQ_GEQ((last->tqe_start + last->tqe_len), th->th_seq))) {
 617                         /* Common case, trailing segment is added */
 618                         /**
 619                          *                                 +--last
 620                          *                                 v
 621                          *  reassembly buffer |---|  |---| |---|
 622                          *  new segment                       |---|
 623                          */
 624 #ifdef TCP_REASS_COUNTERS
 625                         counter_u64_add(reass_path1, 1);
 626 #endif
 627                         if (SEQ_GT((last->tqe_start + last->tqe_len), th->th_seq)) {
 628                                 i = (last->tqe_start + last->tqe_len) - th->th_seq;
 629                                 if (i < *tlenp) {
 630 #ifdef TCP_REASS_LOGGING
 631                                         tcp_log_reassm(tp, last, NULL, 0, i, TCP_R_LOG_TRIM, 3);
 632                                         th->th_seq += i;
 633 #endif
 634                                         m_adj(m, i);
 635                                         *tlenp -= i;
 636                                 } else {
 637                                         /* Complete overlap */
 638                                         TCPSTAT_INC(tcps_rcvduppack);
 639                                         TCPSTAT_ADD(tcps_rcvdupbyte, *tlenp);
 640                                         m_freem(m);
 641                                         *tlenp = last->tqe_len;
 642                                         *seq_start = last->tqe_start;
 643                                         return (0);
 644                                 }
 645                         }
 646                         if (last->tqe_flags & TH_FIN) {
 647                                 /*
 648                                  * We have data after the FIN on the last?
 649                                  */
 650                                 *tlenp = 0;
 651                                 m_freem(m);
 652                                 return(0);
 653                         }
 654                         tcp_reass_append(tp, last, m, th, *tlenp, mlast, lenofoh);
 655                         tp->t_segqmbuflen += lenofoh;
 656                         *seq_start = last->tqe_start;
 657                         *tlenp = last->tqe_len;
 658                         return (0);
 659                 } else if (SEQ_GT(th->th_seq, (last->tqe_start + last->tqe_len))) {
 660                         /*
 661                          * Second common case, we missed
 662                          * another one and have something more
 663                          * for the end.
 664                          */
 665                         /**
 666                          *                                 +--last
 667                          *                                 v
 668                          *  reassembly buffer |---|  |---| |---|
 669                          *  new segment                           |---|
 670                          */
 671                         if (last->tqe_flags & TH_FIN) {
 672                                 /*
 673                                  * We have data after the FIN on the last?
 674                                  */
 675                                 *tlenp = 0;
 676                                 m_freem(m);
 677                                 return(0);
 678                         }
 679 #ifdef TCP_REASS_COUNTERS
 680                         counter_u64_add(reass_path2, 1);
 681 #endif
 682                         p = last;
 683                         goto new_entry;
 684                 }
 685         } else {
 686                 /* First segment (it's NULL). */
 687                 goto new_entry;
 688         }
 689         first = TAILQ_FIRST(&tp->t_segq);
 690         if (SEQ_LT(th->th_seq, first->tqe_start) &&
 691             SEQ_GEQ((th->th_seq + *tlenp),first->tqe_start) &&
 692             SEQ_LT((th->th_seq + *tlenp), (first->tqe_start + first->tqe_len))) {
 693                 /*
 694                  * The head of the queue is prepended by this and
 695                  * it may be the one I want most.
 696                  */
 697                 /**
 698                  *       first-------+
 699                  *                   v
 700                  *  rea:             |---|  |---| |---|
 701                  *  new:         |---|
 702                  * Note the case we do not deal with here is:
 703                  *   rea=     |---|   |---|   |---|
 704                  *   new=  |----|
 705                  * Due to the fact that it could be
 706                  *   new   |--------------------|
 707                  * And we might need to merge forward.
 708                  */
 709 #ifdef INVARIANTS
 710                 struct mbuf *firstmbuf;
 711 #endif
 712
 713 #ifdef TCP_REASS_COUNTERS
 714                 counter_u64_add(reass_path3, 1);
 715 #endif
 716                 if (SEQ_LT(th->th_seq, tp->rcv_nxt)) {
 717                         /*
 718                          * The resend was even before
 719                          * what we have. We need to trim it.
 720                          * Note TSNH (it should be trimmed
 721                          * before the call to tcp_reass()).
 722                          */
 723 #ifdef INVARIANTS
 724                         panic("th->th_seq:%u rcv_nxt:%u tp:%p not pre-trimmed",
 725                               th->th_seq, tp->rcv_nxt, tp);
 726 #else
 727                         i = tp->rcv_nxt - th->th_seq;
 728 #ifdef TCP_REASS_LOGGING
 729                         tcp_log_reassm(tp, first, NULL, 0, i, TCP_R_LOG_TRIM, 4);
 730 #endif
 731                         m_adj(m, i);
 732                         th->th_seq += i;
 733                         *tlenp -= i;
 734 #endif
 735                 }
 736 #ifdef INVARIANTS
 737                 firstmbuf = first->tqe_m;
 738 #endif
 739                 tcp_reass_prepend(tp, first, m, th, *tlenp, mlast, lenofoh);
 740 #ifdef INVARIANTS
 741                 if (firstmbuf == first->tqe_m) {
 742                         panic("First stayed same m:%p foobar:%p first->tqe_m:%p tp:%p first:%p",
 743                               m, firstmbuf, first->tqe_m, tp, first);
 744                 } else if (first->tqe_m != m) {
 745                         panic("First did not change to m:%p foobar:%p first->tqe_m:%p tp:%p first:%p",
 746                               m, firstmbuf, first->tqe_m, tp, first);
 747                 }
 748 #endif
 749                 tp->t_segqmbuflen += lenofoh;
 750                 *seq_start = first->tqe_start;
 751                 *tlenp = first->tqe_len;
 752                 goto present;
 753         } else if (SEQ_LT((th->th_seq + *tlenp), first->tqe_start)) {
 754                 /* New segment is before our earliest segment. */
 755                 /**
 756                  *           first---->+
 757                  *                      v
 758                  *  rea=                |---| ....
 759                  *  new"         |---|
 760                  *
 761                  */
 762                 goto new_entry;
 763         }
 764         /*
 765          * Find a segment which begins after this one does.
 766          */
 767 #ifdef TCP_REASS_COUNTERS
 768         counter_u64_add(reass_fullwalk, 1);
 769 #endif
 770         TAILQ_FOREACH(q, &tp->t_segq, tqe_q) {
 771                 if (SEQ_GT(q->tqe_start, th->th_seq))
 772                         break;
 773         }
 774         p = TAILQ_PREV(q, tsegqe_head, tqe_q);
 775         /**
 776          * Now is this fit just in-between only?
 777          * i.e.:
 778          *      p---+        +----q
 779          *          v        v
 780          *     res= |--|     |--|    |--|
 781          *     nee       |-|
 782          */
 783         if (SEQ_LT((th->th_seq + *tlenp), q->tqe_start) &&
 784             ((p == NULL) || (SEQ_GT(th->th_seq, (p->tqe_start + p->tqe_len))))) {
 785                 /* Yep no overlap */
 786                 goto new_entry;
 787         }
 788         /**
 789          * If we reach here we have some (possibly all) overlap
 790          * such as:
 791          *     res=     |--|     |--|    |--|
 792          *     new=  |----|
 793          * or  new=  |-----------------|
 794          * or  new=      |--------|
 795          * or  new=            |---|
 796          * or  new=            |-----------|
 797          */
 798         if ((p != NULL) &&
 799             (SEQ_LEQ(th->th_seq, (p->tqe_start + p->tqe_len)))) {
 800                 /* conversion to int (in i) handles seq wraparound */
 801
 802 #ifdef TCP_REASS_COUNTERS
 803                 counter_u64_add(reass_path4, 1);
 804 #endif
 805                 i = p->tqe_start + p->tqe_len - th->th_seq;
 806                 if (i >= 0) {
 807                         if (i >= *tlenp) {
 808                                 /**
 809                                  *       prev seg---->+
 810                                  *                    v
 811                                  *  reassembly buffer |---|
 812                                  *  new segment        |-|
 813                                  */
 814                                 TCPSTAT_INC(tcps_rcvduppack);
 815                                 TCPSTAT_ADD(tcps_rcvdupbyte, *tlenp);
 816                                 *tlenp = p->tqe_len;
 817                                 *seq_start = p->tqe_start;
 818                                 m_freem(m);
 819                                 /*
 820                                  * Try to present any queued data
 821                                  * at the left window edge to the user.
 822                                  * This is needed after the 3-WHS
 823                                  * completes. Note this probably
 824                                  * will not work and we will return.
 825                                  */
 826                                 return (0);
 827                         }
 828                         if (i > 0) {
 829                                 /**
 830                                  *       prev seg---->+
 831                                  *                    v
 832                                  *  reassembly buffer |---|
 833                                  *  new segment         |-----|
 834                                  */
 835 #ifdef TCP_REASS_COUNTERS
 836                                 counter_u64_add(reass_path5, 1);
 837 #endif
 838 #ifdef TCP_REASS_LOGGING
 839                                 tcp_log_reassm(tp, p, NULL, 0, i, TCP_R_LOG_TRIM, 5);
 840 #endif
 841                                 m_adj(m, i);
 842                                 *tlenp -= i;
 843                                 th->th_seq += i;
 844                         }
 845                 }
 846                 if (th->th_seq == (p->tqe_start + p->tqe_len)) {
 847                         /*
 848                          * If dovetails in with this one
 849                          * append it.
 850                          */
 851                         /**
 852                          *       prev seg---->+
 853                          *                    v
 854                          *  reassembly buffer |--|     |---|
 855                          *  new segment          |--|
 856                          * (note: it was trimmed above if it overlapped)
 857                          */
 858                         tcp_reass_append(tp, p, m, th, *tlenp, mlast, lenofoh);
 859                         tp->t_segqmbuflen += lenofoh;
 860                 } else {
 861 #ifdef INVARIANTS
 862                         panic("Impossible cut th_seq:%u p->seq:%u(%d) p:%p tp:%p",
 863                               th->th_seq, p->tqe_start, p->tqe_len,
 864                               p, tp);
 865 #endif
 866                         *tlenp = 0;
 867                         m_freem(m);
 868                         return (0);
 869                 }
 870                 q = p;
 871         } else {
 872                 /*
 873                  * The new data runs over the
 874                  * top of previously sack'd data (in q).
 875                  * It may be partially overlapping, or
 876                  * it may overlap the entire segment.
 877                  */
 878 #ifdef TCP_REASS_COUNTERS
 879                 counter_u64_add(reass_path6, 1);
 880 #endif
 881                 if (SEQ_GEQ((th->th_seq + *tlenp), (q->tqe_start + q->tqe_len))) {
 882                         /* It consumes it all */
 883                         /**
 884                          *             next seg---->+
 885                          *                          v
 886                          *  reassembly buffer |--|     |---|
 887                          *  new segment              |----------|
 888                          */
 889 #ifdef TCP_REASS_COUNTERS
 890                         counter_u64_add(reass_path7, 1);
 891 #endif
 892                         tcp_reass_replace(tp, q, m, th->th_seq, *tlenp, mlast, lenofoh, th->th_flags);
 893                 } else {
 894                         /*
 895                          * We just need to prepend the data
 896                          * to this. It does not overrun
 897                          * the end.
 898                          */
 899                         /**
 900                          *                next seg---->+
 901                          *                             v
 902                          *  reassembly buffer |--|     |---|
 903                          *  new segment                   |----------|
 904                          */
 905                         tcp_reass_prepend(tp, q, m, th, *tlenp, mlast, lenofoh);
 906                         tp->t_segqmbuflen += lenofoh;
 907                 }
 908         }
 909         /* Now does it go further than that? */
 910         tcp_reass_merge_forward(tp, q);
 911         *seq_start = q->tqe_start;
 912         *tlenp = q->tqe_len;
 913         goto present;
 914
 915         /*
 916          * When we reach here we can't combine it
 917          * with any existing segment.
 918          *
 919          * Limit the number of segments that can be queued to reduce the
 920          * potential for mbuf exhaustion. For best performance, we want to be
 921          * able to queue a full window's worth of segments. The size of the
 922          * socket receive buffer determines our advertised window and grows
 923          * automatically when socket buffer autotuning is enabled. Use it as the
 924          * basis for our queue limit.
 925          *
 926          * However, allow the user to specify a ceiling for the number of
 927          * segments in each queue.
 928          *
 929          * Always let the missing segment through which caused this queue.
 930          * NB: Access to the socket buffer is left intentionally unlocked as we
 931          * can tolerate stale information here.
 932          *
 933          * XXXLAS: Using sbspace(so->so_rcv) instead of so->so_rcv.sb_hiwat
 934          * should work but causes packets to be dropped when they shouldn't.
 935          * Investigate why and re-evaluate the below limit after the behaviour
 936          * is understood.
 937          */
 938 new_entry:
 939         if (tcp_new_limits) {
 940                 if ((tp->t_segqlen > tcp_reass_queue_guard) &&
 941                     (*tlenp < MSIZE)) {
 942                         /*
 943                          * This is really a lie, we are not full but
 944                          * are getting a segment that is above
 945                          * guard threshold. If it is and its below
 946                          * a mbuf size (256) we drop it if it
 947                          * can't fill in some place.
 948                          */
 949                         TCPSTAT_INC(tcps_rcvreassfull);
 950                         *tlenp = 0;
 951                         if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL, NULL))) {
 952                                 log(LOG_DEBUG, "%s; %s: queue limit reached, "
 953                                     "segment dropped\n", s, __func__);
 954                                 free(s, M_TCPLOG);
 955                         }
 956                         m_freem(m);
 957 #ifdef TCP_REASS_LOGGING
 958                         tcp_reass_log_dump(tp);
 959 #endif
 960                         return (0);
 961                 }
 962         } else {
 963
 964                 if ((th->th_seq != tp->rcv_nxt || !TCPS_HAVEESTABLISHED(tp->t_state)) &&
 965                     tp->t_segqlen >= min((so->so_rcv.sb_hiwat / tp->t_maxseg) + 1,
 966                                          tcp_reass_maxqueuelen)) {
 967                         TCPSTAT_INC(tcps_rcvreassfull);
 968                         *tlenp = 0;
 969                         if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL, NULL))) {
 970                                 log(LOG_DEBUG, "%s; %s: queue limit reached, "
 971                                     "segment dropped\n", s, __func__);
 972                                 free(s, M_TCPLOG);
 973                         }
 974                         m_freem(m);
 975 #ifdef TCP_REASS_LOGGING
 976                         tcp_reass_log_dump(tp);
 977 #endif
 978                         return (0);
 979                 }
 980         }
 981         /*
 982          * Allocate a new queue entry. If we can't, or hit the zone limit
 983          * just drop the pkt.
 984          */
 985         te = uma_zalloc(tcp_reass_zone, M_NOWAIT);
 986         if (te == NULL) {
 987                 TCPSTAT_INC(tcps_rcvmemdrop);
 988                 m_freem(m);
 989                 *tlenp = 0;
 990                 if ((s = tcp_log_addrs(&tp->t_inpcb->inp_inc, th, NULL,
 991                                        NULL))) {
 992                         log(LOG_DEBUG, "%s; %s: global zone limit "
 993                             "reached, segment dropped\n", s, __func__);
 994                         free(s, M_TCPLOG);
 995                 }
 996                 return (0);
 997         }
 998         tp->t_segqlen++;
 999         tp->t_rcvoopack++;
1000         TCPSTAT_INC(tcps_rcvoopack);
1001         TCPSTAT_ADD(tcps_rcvoobyte, *tlenp);
1002         /* Insert the new segment queue entry into place. */
1003         te->tqe_m = m;
1004         te->tqe_flags = th->th_flags;
1005         te->tqe_len = *tlenp;
1006         te->tqe_start = th->th_seq;
1007         te->tqe_last = mlast;
1008         te->tqe_mbuf_cnt = lenofoh;
1009         tp->t_segqmbuflen += te->tqe_mbuf_cnt;
1010         if (p == NULL) {
1011                 TAILQ_INSERT_HEAD(&tp->t_segq, te, tqe_q);
1012         } else {
1013                 TAILQ_INSERT_AFTER(&tp->t_segq, p, te, tqe_q);
1014         }
1015 #ifdef TCP_REASS_LOGGING
1016         tcp_reass_log_new_in(tp, th->th_seq, *tlenp, m, TCP_R_LOG_NEW_ENTRY, te);
1017 #endif
1018 present:
1019         /*
1020          * Present data to user, advancing rcv_nxt through
1021          * completed sequence space.
1022          */
1023         if (!TCPS_HAVEESTABLISHED(tp->t_state))
1024                 return (0);
1025         q = TAILQ_FIRST(&tp->t_segq);
1026         KASSERT(q == NULL || SEQ_GEQ(q->tqe_start, tp->rcv_nxt),
1027                 ("Reassembly queue for %p has stale entry at head", tp));
1028         if (!q || q->tqe_start != tp->rcv_nxt) {
1029 #ifdef TCP_REASS_LOGGING
1030                 tcp_reass_log_dump(tp);
1031 #endif
1032                 return (0);
1033         }
1034         SOCKBUF_LOCK(&so->so_rcv);
1035         do {
1036                 tp->rcv_nxt += q->tqe_len;
1037                 flags = q->tqe_flags & TH_FIN;
1038                 nq = TAILQ_NEXT(q, tqe_q);
1039                 TAILQ_REMOVE(&tp->t_segq, q, tqe_q);
1040                 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
1041                         m_freem(q->tqe_m);
1042                 } else {
1043 #ifdef TCP_REASS_LOGGING
1044                         tcp_reass_log_new_in(tp, q->tqe_start, q->tqe_len, q->tqe_m, TCP_R_LOG_READ, q);
1045                         tcp_log_reassm(tp, q, NULL, th->th_seq, *tlenp, TCP_R_LOG_READ, 1);
1046 #endif
1047                         sbappendstream_locked(&so->so_rcv, q->tqe_m, 0);
1048                 }
1049 #ifdef TCP_REASS_LOGGING
1050                 tcp_log_reassm(tp, q, NULL, th->th_seq, *tlenp, TCP_R_LOG_READ, 2);
1051 #endif
1052                 KASSERT(tp->t_segqmbuflen >= q->tqe_mbuf_cnt,
1053                         ("tp:%p seg queue goes negative", tp));
1054                 tp->t_segqmbuflen -= q->tqe_mbuf_cnt;
1055                 uma_zfree(tcp_reass_zone, q);
1056                 tp->t_segqlen--;
1057                 q = nq;
1058         } while (q && q->tqe_start == tp->rcv_nxt);
1059         if (TAILQ_EMPTY(&tp->t_segq) &&
1060             (tp->t_segqmbuflen != 0)) {
1061 #ifdef INVARIANTS
1062                 panic("tp:%p segq:%p len:%d queue empty",
1063                       tp, &tp->t_segq, tp->t_segqmbuflen);
1064 #else
1065 #ifdef TCP_REASS_LOGGING
1066                 tcp_log_reassm(tp, NULL, NULL, th->th_seq, *tlenp, TCP_R_LOG_ZERO, 0);
1067 #endif
1068                 tp->t_segqmbuflen = 0;
1069 #endif
1070         }
1071 #ifdef TCP_REASS_LOGGING
1072         tcp_reass_log_dump(tp);
1073 #endif
1074         sorwakeup_locked(so);
1075         return (flags);
1076 }