sys/netinet/tcp_lro.c

   1 /******************************************************************************
   2
   3 Copyright (c) 2007, Myricom Inc.
   4 Copyright (c) 2008, Intel Corporation.
   5 All rights reserved.
   6
   7 Redistribution and use in source and binary forms, with or without
   8 modification, are permitted provided that the following conditions are met:
   9
  10  1. Redistributions of source code must retain the above copyright notice,
  11     this list of conditions and the following disclaimer.
  12
  13  2. Neither the name of the Myricom Inc, nor the names of its
  14     contributors may be used to endorse or promote products derived from
  15     this software without specific prior written permission.
  16
  17  3. Neither the name of the Intel Corporation, nor the names of its
  18     contributors may be used to endorse or promote products derived from
  19     this software without specific prior written permission.
  20
  21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  22 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  25 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  31 POSSIBILITY OF SUCH DAMAGE.
  32
  33 $FreeBSD$
  34 ***************************************************************************/
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/endian.h>
  39 #include <sys/mbuf.h>
  40 #include <sys/kernel.h>
  41 #include <sys/socket.h>
  42
  43 #include <net/if.h>
  44 #include <net/ethernet.h>
  45 #include <net/if_media.h>
  46
  47 #include <netinet/in_systm.h>
  48 #include <netinet/in.h>
  49 #include <netinet/ip.h>
  50 #include <netinet/tcp.h>
  51 #include <netinet/tcp_lro.h>
  52
  53 #include <machine/bus.h>
  54 #include <machine/in_cksum.h>
  55
  56
  57 static uint16_t do_csum_data(uint16_t *raw, int len)
  58 {
  59         uint32_t csum;
  60         csum = 0;
  61         while (len > 0) {
  62                 csum += *raw;
  63                 raw++;
  64                 csum += *raw;
  65                 raw++;
  66                 len -= 4;
  67         }
  68         csum = (csum >> 16) + (csum & 0xffff);
  69         csum = (csum >> 16) + (csum & 0xffff);
  70         return (uint16_t)csum;
  71 }
  72
  73 /*
  74  * Allocate and init the LRO data structures
  75  */
  76 int
  77 tcp_lro_init(struct lro_ctrl *cntl)
  78 {
  79         struct lro_entry *lro;
  80         int i, error = 0;
  81
  82         SLIST_INIT(&cntl->lro_free);
  83         SLIST_INIT(&cntl->lro_active);
  84
  85         cntl->lro_bad_csum = 0;
  86         cntl->lro_queued = 0;
  87         cntl->lro_flushed = 0;
  88
  89         for (i = 0; i < LRO_ENTRIES; i++) {
  90                 lro = (struct lro_entry *) malloc(sizeof (struct lro_entry),
  91                     M_DEVBUF, M_NOWAIT | M_ZERO);
  92                 if (lro == NULL) {
  93                         if (i == 0)
  94                                 error = ENOMEM;
  95                         break;
  96                 }
  97                 cntl->lro_cnt = i;
  98                 SLIST_INSERT_HEAD(&cntl->lro_free, lro, next);
  99         }
 100
 101         return (error);
 102 }
 103
 104 void
 105 tcp_lro_free(struct lro_ctrl *cntl)
 106 {
 107         struct lro_entry *entry;
 108
 109         while (!SLIST_EMPTY(&cntl->lro_free)) {
 110                 entry = SLIST_FIRST(&cntl->lro_free);
 111                 SLIST_REMOVE_HEAD(&cntl->lro_free, next);
 112                 free(entry, M_DEVBUF);
 113         }
 114 }
 115
 116 void
 117 tcp_lro_flush(struct lro_ctrl *cntl, struct lro_entry *lro)
 118 {
 119         struct ifnet *ifp;
 120         struct ip *ip;
 121         struct tcphdr *tcp;
 122         uint32_t *ts_ptr;
 123         uint32_t tcplen, tcp_csum;
 124
 125
 126         if (lro->append_cnt) {
 127                 /* incorporate the new len into the ip header and
 128                  * re-calculate the checksum */
 129                 ip = lro->ip;
 130                 ip->ip_len = htons(lro->len - ETHER_HDR_LEN);
 131                 ip->ip_sum = 0;
 132                 ip->ip_sum = 0xffff ^
 133                         do_csum_data((uint16_t*)ip,
 134                                               sizeof (*ip));
 135
 136                 lro->m_head->m_pkthdr.csum_flags = CSUM_IP_CHECKED |
 137                         CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
 138                 lro->m_head->m_pkthdr.csum_data = 0xffff;
 139                 lro->m_head->m_pkthdr.len = lro->len;
 140
 141                 /* incorporate the latest ack into the tcp header */
 142                 tcp = (struct tcphdr *) (ip + 1);
 143                 tcp->th_ack = lro->ack_seq;
 144                 tcp->th_win = lro->window;
 145                 /* incorporate latest timestamp into the tcp header */
 146                 if (lro->timestamp) {
 147                         ts_ptr = (uint32_t *)(tcp + 1);
 148                         ts_ptr[1] = htonl(lro->tsval);
 149                         ts_ptr[2] = lro->tsecr;
 150                 }
 151                 /*
 152                  * update checksum in tcp header by re-calculating the
 153                  * tcp pseudoheader checksum, and adding it to the checksum
 154                  * of the tcp payload data
 155                  */
 156                 tcp->th_sum = 0;
 157                 tcplen = lro->len - sizeof(*ip) - ETHER_HDR_LEN;
 158                 tcp_csum = lro->data_csum;
 159                 tcp_csum += in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
 160                                       htons(tcplen + IPPROTO_TCP));
 161                 tcp_csum += do_csum_data((uint16_t*)tcp,
 162                                                   tcp->th_off << 2);
 163                 tcp_csum = (tcp_csum & 0xffff) + (tcp_csum >> 16);
 164                 tcp_csum = (tcp_csum & 0xffff) + (tcp_csum >> 16);
 165                 tcp->th_sum = 0xffff ^ tcp_csum;
 166         }
 167         ifp = cntl->ifp;
 168         (*ifp->if_input)(cntl->ifp, lro->m_head);
 169         cntl->lro_queued += lro->append_cnt + 1;
 170         cntl->lro_flushed++;
 171         lro->m_head = NULL;
 172         lro->timestamp = 0;
 173         lro->append_cnt = 0;
 174         SLIST_INSERT_HEAD(&cntl->lro_free, lro, next);
 175 }
 176
 177 int
 178 tcp_lro_rx(struct lro_ctrl *cntl, struct mbuf *m_head, uint32_t csum)
 179 {
 180         struct ether_header *eh;
 181         struct ip *ip;
 182         struct tcphdr *tcp;
 183         uint32_t *ts_ptr;
 184         struct mbuf *m_nxt, *m_tail;
 185         struct lro_entry *lro;
 186         int hlen, ip_len, tcp_hdr_len, tcp_data_len, tot_len;
 187         int opt_bytes, trim, csum_flags;
 188         uint32_t seq, tmp_csum, device_mtu;
 189
 190
 191         eh = mtod(m_head, struct ether_header *);
 192         if (eh->ether_type != htons(ETHERTYPE_IP))
 193                 return 1;
 194         ip = (struct ip *) (eh + 1);
 195         if (ip->ip_p != IPPROTO_TCP)
 196                 return 1;
 197
 198         /* ensure there are no options */
 199         if ((ip->ip_hl << 2) != sizeof (*ip))
 200                 return -1;
 201
 202         /* .. and the packet is not fragmented */
 203         if (ip->ip_off & htons(IP_MF|IP_OFFMASK))
 204                 return -1;
 205
 206         /* verify that the IP header checksum is correct */
 207         csum_flags = m_head->m_pkthdr.csum_flags;
 208         if (csum_flags & CSUM_IP_CHECKED) {
 209                 if (__predict_false((csum_flags & CSUM_IP_VALID) == 0)) {
 210                         cntl->lro_bad_csum++;
 211                         return -1;
 212                 }
 213         } else {
 214                 tmp_csum = do_csum_data((uint16_t *)ip, sizeof (*ip));
 215                 if (__predict_false((tmp_csum ^ 0xffff) != 0)) {
 216                         cntl->lro_bad_csum++;
 217                         return -1;
 218                 }
 219         }
 220
 221         /* find the TCP header */
 222         tcp = (struct tcphdr *) (ip + 1);
 223
 224         /* Get the TCP checksum if we dont have it */
 225         if (!csum)
 226                 csum = tcp->th_sum;
 227
 228         /* ensure no bits set besides ack or psh */
 229         if ((tcp->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
 230                 return -1;
 231
 232         /* check for timestamps. Since the only option we handle are
 233            timestamps, we only have to handle the simple case of
 234            aligned timestamps */
 235
 236         opt_bytes = (tcp->th_off << 2) - sizeof (*tcp);
 237         tcp_hdr_len =  sizeof (*tcp) + opt_bytes;
 238         ts_ptr = (uint32_t *)(tcp + 1);
 239         if (opt_bytes != 0) {
 240                 if (__predict_false(opt_bytes != TCPOLEN_TSTAMP_APPA) ||
 241                     (*ts_ptr !=  ntohl(TCPOPT_NOP<<24|TCPOPT_NOP<<16|
 242                     TCPOPT_TIMESTAMP<<8|TCPOLEN_TIMESTAMP)))
 243                         return -1;
 244         }
 245
 246         ip_len = ntohs(ip->ip_len);
 247         tcp_data_len = ip_len - (tcp->th_off << 2) - sizeof (*ip);
 248
 249
 250         /*
 251          * If frame is padded beyond the end of the IP packet,
 252          * then we must trim the extra bytes off the end.
 253          */
 254         tot_len = m_head->m_pkthdr.len;
 255         trim = tot_len - (ip_len + ETHER_HDR_LEN);
 256         if (trim != 0) {
 257                 if (trim < 0) {
 258                         /* truncated packet */
 259                         return -1;
 260                 }
 261                 m_adj(m_head, -trim);
 262                 tot_len = m_head->m_pkthdr.len;
 263         }
 264
 265         m_nxt = m_head;
 266         m_tail = NULL; /* -Wuninitialized */
 267         while (m_nxt != NULL) {
 268                 m_tail = m_nxt;
 269                 m_nxt = m_tail->m_next;
 270         }
 271
 272         hlen = ip_len + ETHER_HDR_LEN - tcp_data_len;
 273         seq = ntohl(tcp->th_seq);
 274
 275         SLIST_FOREACH(lro, &cntl->lro_active, next) {
 276                 if (lro->source_port == tcp->th_sport &&
 277                     lro->dest_port == tcp->th_dport &&
 278                     lro->source_ip == ip->ip_src.s_addr &&
 279                     lro->dest_ip == ip->ip_dst.s_addr) {
 280                         /* Flush now if appending will result in overflow. */
 281                         if (lro->len > (65535 - tcp_data_len)) {
 282                                 SLIST_REMOVE(&cntl->lro_active, lro,
 283                                              lro_entry, next);
 284                                 tcp_lro_flush(cntl, lro);
 285                                 break;
 286                         }
 287
 288                         /* Try to append it */
 289
 290                         if (__predict_false(seq != lro->next_seq ||
 291                                     (tcp_data_len == 0 &&
 292                                     lro->ack_seq == tcp->th_ack))) {
 293                                 /* out of order packet or dup ack */
 294                                 SLIST_REMOVE(&cntl->lro_active, lro,
 295                                              lro_entry, next);
 296                                 tcp_lro_flush(cntl, lro);
 297                                 return -1;
 298                         }
 299
 300                         if (opt_bytes) {
 301                                 uint32_t tsval = ntohl(*(ts_ptr + 1));
 302                                 /* make sure timestamp values are increasing */
 303                                 if (__predict_false(lro->tsval > tsval ||
 304                                              *(ts_ptr + 2) == 0)) {
 305                                         return -1;
 306                                 }
 307                                 lro->tsval = tsval;
 308                                 lro->tsecr = *(ts_ptr + 2);
 309                         }
 310
 311                         lro->next_seq += tcp_data_len;
 312                         lro->ack_seq = tcp->th_ack;
 313                         lro->window = tcp->th_win;
 314                         lro->append_cnt++;
 315                         if (tcp_data_len == 0) {
 316                                 m_freem(m_head);
 317                                 return 0;
 318                         }
 319                         /* subtract off the checksum of the tcp header
 320                          * from the hardware checksum, and add it to the
 321                          * stored tcp data checksum.  Byteswap the checksum
 322                          * if the total length so far is odd
 323                          */
 324                         tmp_csum = do_csum_data((uint16_t*)tcp,
 325                                                          tcp_hdr_len);
 326                         csum = csum + (tmp_csum ^ 0xffff);
 327                         csum = (csum & 0xffff) + (csum >> 16);
 328                         csum = (csum & 0xffff) + (csum >> 16);
 329                         if (lro->len & 0x1) {
 330                                 /* Odd number of bytes so far, flip bytes */
 331                                 csum = ((csum << 8) | (csum >> 8)) & 0xffff;
 332                         }
 333                         csum = csum + lro->data_csum;
 334                         csum = (csum & 0xffff) + (csum >> 16);
 335                         csum = (csum & 0xffff) + (csum >> 16);
 336                         lro->data_csum = csum;
 337
 338                         lro->len += tcp_data_len;
 339
 340                         /* adjust mbuf so that m->m_data points to
 341                            the first byte of the payload */
 342                         m_adj(m_head, hlen);
 343                         /* append mbuf chain */
 344                         lro->m_tail->m_next = m_head;
 345                         /* advance the last pointer */
 346                         lro->m_tail = m_tail;
 347                         /* flush packet if required */
 348                         device_mtu = cntl->ifp->if_mtu;
 349                         if (lro->len > (65535 - device_mtu)) {
 350                                 SLIST_REMOVE(&cntl->lro_active, lro,
 351                                              lro_entry, next);
 352                                 tcp_lro_flush(cntl, lro);
 353                         }
 354                         return 0;
 355                 }
 356         }
 357
 358         if (SLIST_EMPTY(&cntl->lro_free))
 359             return -1;
 360
 361         /* start a new chain */
 362         lro = SLIST_FIRST(&cntl->lro_free);
 363         SLIST_REMOVE_HEAD(&cntl->lro_free, next);
 364         SLIST_INSERT_HEAD(&cntl->lro_active, lro, next);
 365         lro->source_port = tcp->th_sport;
 366         lro->dest_port = tcp->th_dport;
 367         lro->source_ip = ip->ip_src.s_addr;
 368         lro->dest_ip = ip->ip_dst.s_addr;
 369         lro->next_seq = seq + tcp_data_len;
 370         lro->mss = tcp_data_len;
 371         lro->ack_seq = tcp->th_ack;
 372         lro->window = tcp->th_win;
 373
 374         /* save the checksum of just the TCP payload by
 375          * subtracting off the checksum of the TCP header from
 376          * the entire hardware checksum
 377          * Since IP header checksum is correct, checksum over
 378          * the IP header is -0.  Substracting -0 is unnecessary.
 379          */
 380         tmp_csum = do_csum_data((uint16_t*)tcp, tcp_hdr_len);
 381         csum = csum + (tmp_csum ^ 0xffff);
 382         csum = (csum & 0xffff) + (csum >> 16);
 383         csum = (csum & 0xffff) + (csum >> 16);
 384         lro->data_csum = csum;
 385
 386         lro->ip = ip;
 387         /* record timestamp if it is present */
 388         if (opt_bytes) {
 389                 lro->timestamp = 1;
 390                 lro->tsval = ntohl(*(ts_ptr + 1));
 391                 lro->tsecr = *(ts_ptr + 2);
 392         }
 393         lro->len = tot_len;
 394         lro->m_head = m_head;
 395         lro->m_tail = m_tail;
 396         return 0;
 397 }