2 * Copyright (c) 2007, Myricom Inc.
3 * Copyright (c) 2008, Intel Corporation.
4 * Copyright (c) 2012 The FreeBSD Foundation
5 * Copyright (c) 2016 Mellanox Technologies.
8 * Portions of this software were developed by Bjoern Zeeb
9 * under sponsorship from the FreeBSD Foundation.
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
37 #include "opt_inet6.h"
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/malloc.h>
44 #include <sys/socket.h>
47 #include <net/if_var.h>
48 #include <net/ethernet.h>
51 #include <netinet/in_systm.h>
52 #include <netinet/in.h>
53 #include <netinet/ip6.h>
54 #include <netinet/ip.h>
55 #include <netinet/ip_var.h>
56 #include <netinet/tcp.h>
57 #include <netinet/tcp_lro.h>
59 #include <netinet6/ip6_var.h>
61 #include <machine/in_cksum.h>
63 static MALLOC_DEFINE(M_LRO, "LRO", "LRO control structures");
65 #define TCP_LRO_UPDATE_CSUM 1
66 #ifndef TCP_LRO_UPDATE_CSUM
67 #define TCP_LRO_INVALID_CSUM 0x0000
70 static void tcp_lro_rx_done(struct lro_ctrl *lc);
73 tcp_lro_active_insert(struct lro_ctrl *lc, struct lro_entry *le)
76 LIST_INSERT_HEAD(&lc->lro_active, le, next);
80 tcp_lro_active_remove(struct lro_entry *le)
83 LIST_REMOVE(le, next);
87 tcp_lro_init(struct lro_ctrl *lc)
89 return (tcp_lro_init_args(lc, NULL, TCP_LRO_ENTRIES, 0));
93 tcp_lro_init_args(struct lro_ctrl *lc, struct ifnet *ifp,
94 unsigned lro_entries, unsigned lro_mbufs)
100 lc->lro_bad_csum = 0;
104 lc->lro_mbuf_count = 0;
105 lc->lro_mbuf_max = lro_mbufs;
106 lc->lro_cnt = lro_entries;
107 lc->lro_ackcnt_lim = TCP_LRO_ACKCNT_MAX;
108 lc->lro_length_lim = TCP_LRO_LENGTH_MAX;
110 LIST_INIT(&lc->lro_free);
111 LIST_INIT(&lc->lro_active);
113 /* compute size to allocate */
114 size = (lro_mbufs * sizeof(struct lro_mbuf_sort)) +
115 (lro_entries * sizeof(*le));
116 lc->lro_mbuf_data = (struct lro_mbuf_sort *)
117 malloc(size, M_LRO, M_NOWAIT | M_ZERO);
119 /* check for out of memory */
120 if (lc->lro_mbuf_data == NULL) {
121 memset(lc, 0, sizeof(*lc));
124 /* compute offset for LRO entries */
125 le = (struct lro_entry *)
126 (lc->lro_mbuf_data + lro_mbufs);
128 /* setup linked list */
129 for (i = 0; i != lro_entries; i++)
130 LIST_INSERT_HEAD(&lc->lro_free, le + i, next);
136 tcp_lro_free(struct lro_ctrl *lc)
138 struct lro_entry *le;
141 /* reset LRO free list */
142 LIST_INIT(&lc->lro_free);
144 /* free active mbufs, if any */
145 while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {
146 tcp_lro_active_remove(le);
150 /* free mbuf array, if any */
151 for (x = 0; x != lc->lro_mbuf_count; x++)
152 m_freem(lc->lro_mbuf_data[x].mb);
153 lc->lro_mbuf_count = 0;
155 /* free allocated memory, if any */
156 free(lc->lro_mbuf_data, M_LRO);
157 lc->lro_mbuf_data = NULL;
160 #ifdef TCP_LRO_UPDATE_CSUM
162 tcp_lro_csum_th(struct tcphdr *th)
167 ch = th->th_sum = 0x0000;
178 ch = (ch >> 16) + (ch & 0xffff);
180 return (ch & 0xffff);
184 tcp_lro_rx_csum_fixup(struct lro_entry *le, void *l3hdr, struct tcphdr *th,
185 uint16_t tcp_data_len, uint16_t csum)
192 /* Remove length from checksum. */
193 switch (le->eh_type) {
199 ip6 = (struct ip6_hdr *)l3hdr;
200 if (le->append_cnt == 0)
205 cx = ntohs(ip6->ip6_plen);
206 cs = in6_cksum_pseudo(ip6, cx, ip6->ip6_nxt, 0);
216 ip4 = (struct ip *)l3hdr;
217 if (le->append_cnt == 0)
220 cs = in_addword(ntohs(ip4->ip_len) - sizeof(*ip4),
222 cs = in_pseudo(ip4->ip_src.s_addr, ip4->ip_dst.s_addr,
229 cs = 0; /* Keep compiler happy. */
235 /* Remove TCP header csum. */
236 cs = ~tcp_lro_csum_th(th);
239 c = (c >> 16) + (c & 0xffff);
246 tcp_lro_rx_done(struct lro_ctrl *lc)
248 struct lro_entry *le;
250 while ((le = LIST_FIRST(&lc->lro_active)) != NULL) {
251 tcp_lro_active_remove(le);
252 tcp_lro_flush(lc, le);
257 tcp_lro_flush_inactive(struct lro_ctrl *lc, const struct timeval *timeout)
259 struct lro_entry *le, *le_tmp;
262 if (LIST_EMPTY(&lc->lro_active))
266 timevalsub(&tv, timeout);
267 LIST_FOREACH_SAFE(le, &lc->lro_active, next, le_tmp) {
268 if (timevalcmp(&tv, &le->mtime, >=)) {
269 tcp_lro_active_remove(le);
270 tcp_lro_flush(lc, le);
276 tcp_lro_flush(struct lro_ctrl *lc, struct lro_entry *le)
279 if (le->append_cnt > 0) {
283 p_len = htons(le->p_len);
284 switch (le->eh_type) {
291 ip6->ip6_plen = p_len;
292 th = (struct tcphdr *)(ip6 + 1);
293 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |
295 le->p_len += ETHER_HDR_LEN + sizeof(*ip6);
303 #ifdef TCP_LRO_UPDATE_CSUM
309 #ifdef TCP_LRO_UPDATE_CSUM
310 /* Fix IP header checksum for new length. */
316 cl = (cl >> 16) + (cl & 0xffff);
320 ip4->ip_sum = TCP_LRO_INVALID_CSUM;
323 th = (struct tcphdr *)(ip4 + 1);
324 le->m_head->m_pkthdr.csum_flags = CSUM_DATA_VALID |
325 CSUM_PSEUDO_HDR | CSUM_IP_CHECKED | CSUM_IP_VALID;
326 le->p_len += ETHER_HDR_LEN;
331 th = NULL; /* Keep compiler happy. */
333 le->m_head->m_pkthdr.csum_data = 0xffff;
334 le->m_head->m_pkthdr.len = le->p_len;
336 /* Incorporate the latest ACK into the TCP header. */
337 th->th_ack = le->ack_seq;
338 th->th_win = le->window;
339 /* Incorporate latest timestamp into the TCP header. */
340 if (le->timestamp != 0) {
343 ts_ptr = (uint32_t *)(th + 1);
344 ts_ptr[1] = htonl(le->tsval);
345 ts_ptr[2] = le->tsecr;
347 #ifdef TCP_LRO_UPDATE_CSUM
348 /* Update the TCP header checksum. */
349 le->ulp_csum += p_len;
350 le->ulp_csum += tcp_lro_csum_th(th);
351 while (le->ulp_csum > 0xffff)
352 le->ulp_csum = (le->ulp_csum >> 16) +
353 (le->ulp_csum & 0xffff);
354 th->th_sum = (le->ulp_csum & 0xffff);
355 th->th_sum = ~th->th_sum;
357 th->th_sum = TCP_LRO_INVALID_CSUM;
361 (*lc->ifp->if_input)(lc->ifp, le->m_head);
362 lc->lro_queued += le->append_cnt + 1;
364 bzero(le, sizeof(*le));
365 LIST_INSERT_HEAD(&lc->lro_free, le, next);
368 #ifdef HAVE_INLINE_FLSLL
369 #define tcp_lro_msb_64(x) (1ULL << (flsll(x) - 1))
371 static inline uint64_t
372 tcp_lro_msb_64(uint64_t x)
380 return (x & ~(x >> 1));
385 * The tcp_lro_sort() routine is comparable to qsort(), except it has
386 * a worst case complexity limit of O(MIN(N,64)*N), where N is the
387 * number of elements to sort and 64 is the number of sequence bits
388 * available. The algorithm is bit-slicing the 64-bit sequence number,
389 * sorting one bit at a time from the most significant bit until the
390 * least significant one, skipping the constant bits. This is
391 * typically called a radix sort.
394 tcp_lro_sort(struct lro_mbuf_sort *parray, uint32_t size)
396 struct lro_mbuf_sort temp;
403 /* for small arrays insertion sort is faster */
405 for (x = 1; x < size; x++) {
407 for (y = x; y > 0 && temp.seq < parray[y - 1].seq; y--)
408 parray[y] = parray[y - 1];
414 /* compute sequence bits which are constant */
417 for (x = 0; x != size; x++) {
418 ones |= parray[x].seq;
419 zeros |= ~parray[x].seq;
422 /* compute bits which are not constant into "ones" */
427 /* pick the most significant bit which is not constant */
428 ones = tcp_lro_msb_64(ones);
431 * Move entries having cleared sequence bits to the beginning
434 for (x = y = 0; y != size; y++) {
436 if (parray[y].seq & ones)
440 parray[x] = parray[y];
445 KASSERT(x != 0 && x != size, ("Memory is corrupted\n"));
448 tcp_lro_sort(parray, x);
457 tcp_lro_flush_all(struct lro_ctrl *lc)
463 /* check if no mbufs to flush */
464 if (lc->lro_mbuf_count == 0)
467 /* sort all mbufs according to stream */
468 tcp_lro_sort(lc->lro_mbuf_data, lc->lro_mbuf_count);
470 /* input data into LRO engine, stream by stream */
472 for (x = 0; x != lc->lro_mbuf_count; x++) {
476 mb = lc->lro_mbuf_data[x].mb;
478 /* get sequence number, masking away the packet index */
479 nseq = lc->lro_mbuf_data[x].seq & (-1ULL << 24);
481 /* check for new stream */
485 /* flush active streams */
489 /* add packet to LRO engine */
490 if (tcp_lro_rx(lc, mb, 0) != 0) {
491 /* input packet to network layer */
492 (*lc->ifp->if_input)(lc->ifp, mb);
498 /* flush active streams */
501 lc->lro_mbuf_count = 0;
506 tcp_lro_rx_ipv6(struct lro_ctrl *lc, struct mbuf *m, struct ip6_hdr *ip6,
510 /* XXX-BZ we should check the flow-label. */
512 /* XXX-BZ We do not yet support ext. hdrs. */
513 if (ip6->ip6_nxt != IPPROTO_TCP)
514 return (TCP_LRO_NOT_SUPPORTED);
516 /* Find the TCP header. */
517 *th = (struct tcphdr *)(ip6 + 1);
525 tcp_lro_rx_ipv4(struct lro_ctrl *lc, struct mbuf *m, struct ip *ip4,
531 if (ip4->ip_p != IPPROTO_TCP)
532 return (TCP_LRO_NOT_SUPPORTED);
534 /* Ensure there are no options. */
535 if ((ip4->ip_hl << 2) != sizeof (*ip4))
536 return (TCP_LRO_CANNOT);
538 /* .. and the packet is not fragmented. */
539 if (ip4->ip_off & htons(IP_MF|IP_OFFMASK))
540 return (TCP_LRO_CANNOT);
542 /* Legacy IP has a header checksum that needs to be correct. */
543 csum_flags = m->m_pkthdr.csum_flags;
544 if (csum_flags & CSUM_IP_CHECKED) {
545 if (__predict_false((csum_flags & CSUM_IP_VALID) == 0)) {
547 return (TCP_LRO_CANNOT);
550 csum = in_cksum_hdr(ip4);
551 if (__predict_false((csum) != 0)) {
553 return (TCP_LRO_CANNOT);
557 /* Find the TCP header (we assured there are no IP options). */
558 *th = (struct tcphdr *)(ip4 + 1);
565 tcp_lro_rx(struct lro_ctrl *lc, struct mbuf *m, uint32_t csum)
567 struct lro_entry *le;
568 struct ether_header *eh;
570 struct ip6_hdr *ip6 = NULL; /* Keep compiler happy. */
573 struct ip *ip4 = NULL; /* Keep compiler happy. */
576 void *l3hdr = NULL; /* Keep compiler happy. */
579 int error, ip_len, l;
580 uint16_t eh_type, tcp_data_len;
582 /* We expect a contiguous header [eh, ip, tcp]. */
584 eh = mtod(m, struct ether_header *);
585 eh_type = ntohs(eh->ether_type);
590 CURVNET_SET(lc->ifp->if_vnet);
591 if (V_ip6_forwarding != 0) {
592 /* XXX-BZ stats but changing lro_ctrl is a problem. */
594 return (TCP_LRO_CANNOT);
597 l3hdr = ip6 = (struct ip6_hdr *)(eh + 1);
598 error = tcp_lro_rx_ipv6(lc, m, ip6, &th);
601 tcp_data_len = ntohs(ip6->ip6_plen);
602 ip_len = sizeof(*ip6) + tcp_data_len;
609 CURVNET_SET(lc->ifp->if_vnet);
610 if (V_ipforwarding != 0) {
611 /* XXX-BZ stats but changing lro_ctrl is a problem. */
613 return (TCP_LRO_CANNOT);
616 l3hdr = ip4 = (struct ip *)(eh + 1);
617 error = tcp_lro_rx_ipv4(lc, m, ip4, &th);
620 ip_len = ntohs(ip4->ip_len);
621 tcp_data_len = ip_len - sizeof(*ip4);
625 /* XXX-BZ what happens in case of VLAN(s)? */
627 return (TCP_LRO_NOT_SUPPORTED);
631 * If the frame is padded beyond the end of the IP packet, then we must
632 * trim the extra bytes off.
634 l = m->m_pkthdr.len - (ETHER_HDR_LEN + ip_len);
637 /* Truncated packet. */
638 return (TCP_LRO_CANNOT);
644 * Check TCP header constraints.
646 /* Ensure no bits set besides ACK or PSH. */
647 if ((th->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
648 return (TCP_LRO_CANNOT);
650 /* XXX-BZ We lose a ACK|PUSH flag concatenating multiple segments. */
651 /* XXX-BZ Ideally we'd flush on PUSH? */
654 * Check for timestamps.
655 * Since the only option we handle are timestamps, we only have to
656 * handle the simple case of aligned timestamps.
658 l = (th->th_off << 2);
661 ts_ptr = (uint32_t *)(th + 1);
662 if (l != 0 && (__predict_false(l != TCPOLEN_TSTAMP_APPA) ||
663 (*ts_ptr != ntohl(TCPOPT_NOP<<24|TCPOPT_NOP<<16|
664 TCPOPT_TIMESTAMP<<8|TCPOLEN_TIMESTAMP))))
665 return (TCP_LRO_CANNOT);
667 /* If the driver did not pass in the checksum, set it now. */
671 seq = ntohl(th->th_seq);
673 /* Try to find a matching previous segment. */
674 LIST_FOREACH(le, &lc->lro_active, next) {
675 if (le->eh_type != eh_type)
677 if (le->source_port != th->th_sport ||
678 le->dest_port != th->th_dport)
683 if (bcmp(&le->source_ip6, &ip6->ip6_src,
684 sizeof(struct in6_addr)) != 0 ||
685 bcmp(&le->dest_ip6, &ip6->ip6_dst,
686 sizeof(struct in6_addr)) != 0)
692 if (le->source_ip4 != ip4->ip_src.s_addr ||
693 le->dest_ip4 != ip4->ip_dst.s_addr)
699 /* Flush now if appending will result in overflow. */
700 if (le->p_len > (lc->lro_length_lim - tcp_data_len)) {
701 tcp_lro_active_remove(le);
702 tcp_lro_flush(lc, le);
706 /* Try to append the new segment. */
707 if (__predict_false(seq != le->next_seq ||
708 (tcp_data_len == 0 && le->ack_seq == th->th_ack))) {
709 /* Out of order packet or duplicate ACK. */
710 tcp_lro_active_remove(le);
711 tcp_lro_flush(lc, le);
712 return (TCP_LRO_CANNOT);
716 uint32_t tsval = ntohl(*(ts_ptr + 1));
717 /* Make sure timestamp values are increasing. */
718 /* XXX-BZ flip and use TSTMP_GEQ macro for this? */
719 if (__predict_false(le->tsval > tsval ||
721 return (TCP_LRO_CANNOT);
723 le->tsecr = *(ts_ptr + 2);
726 le->next_seq += tcp_data_len;
727 le->ack_seq = th->th_ack;
728 le->window = th->th_win;
731 #ifdef TCP_LRO_UPDATE_CSUM
732 le->ulp_csum += tcp_lro_rx_csum_fixup(le, l3hdr, th,
733 tcp_data_len, ~csum);
736 if (tcp_data_len == 0) {
739 * Flush this LRO entry, if this ACK should not
740 * be further delayed.
742 if (le->append_cnt >= lc->lro_ackcnt_lim) {
743 tcp_lro_active_remove(le);
744 tcp_lro_flush(lc, le);
749 le->p_len += tcp_data_len;
752 * Adjust the mbuf so that m_data points to the first byte of
753 * the ULP payload. Adjust the mbuf to avoid complications and
754 * append new segment to existing mbuf chain.
756 m_adj(m, m->m_pkthdr.len - tcp_data_len);
759 le->m_tail->m_next = m;
760 le->m_tail = m_last(m);
763 * If a possible next full length packet would cause an
764 * overflow, pro-actively flush now.
766 if (le->p_len > (lc->lro_length_lim - lc->ifp->if_mtu)) {
767 tcp_lro_active_remove(le);
768 tcp_lro_flush(lc, le);
770 getmicrotime(&le->mtime);
775 /* Try to find an empty slot. */
776 if (LIST_EMPTY(&lc->lro_free))
777 return (TCP_LRO_NO_ENTRIES);
779 /* Start a new segment chain. */
780 le = LIST_FIRST(&lc->lro_free);
781 LIST_REMOVE(le, next);
782 tcp_lro_active_insert(lc, le);
783 getmicrotime(&le->mtime);
785 /* Start filling in details. */
790 le->source_ip6 = ip6->ip6_src;
791 le->dest_ip6 = ip6->ip6_dst;
792 le->eh_type = eh_type;
793 le->p_len = m->m_pkthdr.len - ETHER_HDR_LEN - sizeof(*ip6);
799 le->source_ip4 = ip4->ip_src.s_addr;
800 le->dest_ip4 = ip4->ip_dst.s_addr;
801 le->eh_type = eh_type;
802 le->p_len = m->m_pkthdr.len - ETHER_HDR_LEN;
806 le->source_port = th->th_sport;
807 le->dest_port = th->th_dport;
809 le->next_seq = seq + tcp_data_len;
810 le->ack_seq = th->th_ack;
811 le->window = th->th_win;
814 le->tsval = ntohl(*(ts_ptr + 1));
815 le->tsecr = *(ts_ptr + 2);
818 #ifdef TCP_LRO_UPDATE_CSUM
820 * Do not touch the csum of the first packet. However save the
821 * "adjusted" checksum of just the source and destination addresses,
822 * the next header and the TCP payload. The length and TCP header
823 * parts may change, so we remove those from the saved checksum and
824 * re-add with final values on tcp_lro_flush() if needed.
826 KASSERT(le->ulp_csum == 0, ("%s: le=%p le->ulp_csum=0x%04x\n",
827 __func__, le, le->ulp_csum));
829 le->ulp_csum = tcp_lro_rx_csum_fixup(le, l3hdr, th, tcp_data_len,
831 th->th_sum = csum; /* Restore checksum on first packet. */
835 le->m_tail = m_last(m);
841 tcp_lro_queue_mbuf(struct lro_ctrl *lc, struct mbuf *mb)
844 if (__predict_false(lc->ifp == NULL || lc->lro_mbuf_data == NULL ||
845 lc->lro_mbuf_max == 0)) {
851 /* check if packet is not LRO capable */
852 if (__predict_false(mb->m_pkthdr.csum_flags == 0 ||
853 (lc->ifp->if_capenable & IFCAP_LRO) == 0)) {
857 /* input packet to network layer */
858 (*lc->ifp->if_input) (lc->ifp, mb);
862 /* check if array is full */
863 if (__predict_false(lc->lro_mbuf_count == lc->lro_mbuf_max))
864 tcp_lro_flush_all(lc);
866 /* create sequence number */
867 lc->lro_mbuf_data[lc->lro_mbuf_count].seq =
868 (((uint64_t)M_HASHTYPE_GET(mb)) << 56) |
869 (((uint64_t)mb->m_pkthdr.flowid) << 24) |
870 ((uint64_t)lc->lro_mbuf_count);
873 lc->lro_mbuf_data[lc->lro_mbuf_count++].mb = mb;