sys/dev/netmap/netmap_offloadings.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (C) 2014-2015 Vincenzo Maffione
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *   1. Redistributions of source code must retain the above copyright
  11  *      notice, this list of conditions and the following disclaimer.
  12  *   2. Redistributions in binary form must reproduce the above copyright
  13  *      notice, this list of conditions and the following disclaimer in the
  14  *      documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 /* $FreeBSD$ */
  30
  31 #if defined(__FreeBSD__)
  32 #include <sys/cdefs.h> /* prerequisite */
  33
  34 #include <sys/types.h>
  35 #include <sys/errno.h>
  36 #include <sys/param.h>  /* defines used in kernel.h */
  37 #include <sys/kernel.h> /* types used in module initialization */
  38 #include <sys/sockio.h>
  39 #include <sys/malloc.h>
  40 #include <sys/socketvar.h>      /* struct socket */
  41 #include <sys/socket.h> /* sockaddrs */
  42 #include <net/if.h>
  43 #include <net/if_var.h>
  44 #include <machine/bus.h>        /* bus_dmamap_* */
  45 #include <sys/endian.h>
  46
  47 #elif defined(linux)
  48
  49 #include "bsd_glue.h"
  50
  51 #elif defined(__APPLE__)
  52
  53 #warning OSX support is only partial
  54 #include "osx_glue.h"
  55
  56 #else
  57
  58 #error  Unsupported platform
  59
  60 #endif /* unsupported */
  61
  62 #include <net/netmap.h>
  63 #include <dev/netmap/netmap_kern.h>
  64
  65
  66
  67 /* This routine is called by bdg_mismatch_datapath() when it finishes
  68  * accumulating bytes for a segment, in order to fix some fields in the
  69  * segment headers (which still contain the same content as the header
  70  * of the original GSO packet). 'pkt' points to the beginning of the IP
  71  * header of the segment, while 'len' is the length of the IP packet.
  72  */
  73 static void
  74 gso_fix_segment(uint8_t *pkt, size_t len, u_int ipv4, u_int iphlen, u_int tcp,
  75                 u_int idx, u_int segmented_bytes, u_int last_segment)
  76 {
  77         struct nm_iphdr *iph = (struct nm_iphdr *)(pkt);
  78         struct nm_ipv6hdr *ip6h = (struct nm_ipv6hdr *)(pkt);
  79         uint16_t *check = NULL;
  80         uint8_t *check_data = NULL;
  81
  82         if (ipv4) {
  83                 /* Set the IPv4 "Total Length" field. */
  84                 iph->tot_len = htobe16(len);
  85                 nm_prdis("ip total length %u", be16toh(ip->tot_len));
  86
  87                 /* Set the IPv4 "Identification" field. */
  88                 iph->id = htobe16(be16toh(iph->id) + idx);
  89                 nm_prdis("ip identification %u", be16toh(iph->id));
  90
  91                 /* Compute and insert the IPv4 header checksum. */
  92                 iph->check = 0;
  93                 iph->check = nm_os_csum_ipv4(iph);
  94                 nm_prdis("IP csum %x", be16toh(iph->check));
  95         } else {
  96                 /* Set the IPv6 "Payload Len" field. */
  97                 ip6h->payload_len = htobe16(len-iphlen);
  98         }
  99
 100         if (tcp) {
 101                 struct nm_tcphdr *tcph = (struct nm_tcphdr *)(pkt + iphlen);
 102
 103                 /* Set the TCP sequence number. */
 104                 tcph->seq = htobe32(be32toh(tcph->seq) + segmented_bytes);
 105                 nm_prdis("tcp seq %u", be32toh(tcph->seq));
 106
 107                 /* Zero the PSH and FIN TCP flags if this is not the last
 108                    segment. */
 109                 if (!last_segment)
 110                         tcph->flags &= ~(0x8 | 0x1);
 111                 nm_prdis("last_segment %u", last_segment);
 112
 113                 check = &tcph->check;
 114                 check_data = (uint8_t *)tcph;
 115         } else { /* UDP */
 116                 struct nm_udphdr *udph = (struct nm_udphdr *)(pkt + iphlen);
 117
 118                 /* Set the UDP 'Length' field. */
 119                 udph->len = htobe16(len-iphlen);
 120
 121                 check = &udph->check;
 122                 check_data = (uint8_t *)udph;
 123         }
 124
 125         /* Compute and insert TCP/UDP checksum. */
 126         *check = 0;
 127         if (ipv4)
 128                 nm_os_csum_tcpudp_ipv4(iph, check_data, len-iphlen, check);
 129         else
 130                 nm_os_csum_tcpudp_ipv6(ip6h, check_data, len-iphlen, check);
 131
 132         nm_prdis("TCP/UDP csum %x", be16toh(*check));
 133 }
 134
 135 static inline int
 136 vnet_hdr_is_bad(struct nm_vnet_hdr *vh)
 137 {
 138         uint8_t gso_type = vh->gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
 139
 140         return (
 141                 (gso_type != VIRTIO_NET_HDR_GSO_NONE &&
 142                  gso_type != VIRTIO_NET_HDR_GSO_TCPV4 &&
 143                  gso_type != VIRTIO_NET_HDR_GSO_UDP &&
 144                  gso_type != VIRTIO_NET_HDR_GSO_TCPV6)
 145                 ||
 146                  (vh->flags & ~(VIRTIO_NET_HDR_F_NEEDS_CSUM
 147                                | VIRTIO_NET_HDR_F_DATA_VALID))
 148                );
 149 }
 150
 151 /* The VALE mismatch datapath implementation. */
 152 void
 153 bdg_mismatch_datapath(struct netmap_vp_adapter *na,
 154                       struct netmap_vp_adapter *dst_na,
 155                       const struct nm_bdg_fwd *ft_p,
 156                       struct netmap_ring *dst_ring,
 157                       u_int *j, u_int lim, u_int *howmany)
 158 {
 159         struct netmap_slot *dst_slot = NULL;
 160         struct nm_vnet_hdr *vh = NULL;
 161         const struct nm_bdg_fwd *ft_end = ft_p + ft_p->ft_frags;
 162
 163         /* Source and destination pointers. */
 164         uint8_t *dst, *src;
 165         size_t src_len, dst_len;
 166
 167         /* Indices and counters for the destination ring. */
 168         u_int j_start = *j;
 169         u_int j_cur = j_start;
 170         u_int dst_slots = 0;
 171
 172         if (unlikely(ft_p == ft_end)) {
 173                 nm_prlim(1, "No source slots to process");
 174                 return;
 175         }
 176
 177         /* Init source and dest pointers. */
 178         src = ft_p->ft_buf;
 179         src_len = ft_p->ft_len;
 180         dst_slot = &dst_ring->slot[j_cur];
 181         dst = NMB(&dst_na->up, dst_slot);
 182         dst_len = src_len;
 183
 184         /* If the source port uses the offloadings, while destination doesn't,
 185          * we grab the source virtio-net header and do the offloadings here.
 186          */
 187         if (na->up.virt_hdr_len && !dst_na->up.virt_hdr_len) {
 188                 vh = (struct nm_vnet_hdr *)src;
 189                 /* Initial sanity check on the source virtio-net header. If
 190                  * something seems wrong, just drop the packet. */
 191                 if (src_len < na->up.virt_hdr_len) {
 192                         nm_prlim(1, "Short src vnet header, dropping");
 193                         return;
 194                 }
 195                 if (unlikely(vnet_hdr_is_bad(vh))) {
 196                         nm_prlim(1, "Bad src vnet header, dropping");
 197                         return;
 198                 }
 199         }
 200
 201         /* We are processing the first input slot and there is a mismatch
 202          * between source and destination virt_hdr_len (SHL and DHL).
 203          * When the a client is using virtio-net headers, the header length
 204          * can be:
 205          *    - 10: the header corresponds to the struct nm_vnet_hdr
 206          *    - 12: the first 10 bytes correspond to the struct
 207          *          virtio_net_hdr, and the last 2 bytes store the
 208          *          "mergeable buffers" info, which is an optional
 209          *          hint that can be zeroed for compatibility
 210          *
 211          * The destination header is therefore built according to the
 212          * following table:
 213          *
 214          * SHL | DHL | destination header
 215          * -----------------------------
 216          *   0 |  10 | zero
 217          *   0 |  12 | zero
 218          *  10 |   0 | doesn't exist
 219          *  10 |  12 | first 10 bytes are copied from source header, last 2 are zero
 220          *  12 |   0 | doesn't exist
 221          *  12 |  10 | copied from the first 10 bytes of source header
 222          */
 223         bzero(dst, dst_na->up.virt_hdr_len);
 224         if (na->up.virt_hdr_len && dst_na->up.virt_hdr_len)
 225                 memcpy(dst, src, sizeof(struct nm_vnet_hdr));
 226         /* Skip the virtio-net headers. */
 227         src += na->up.virt_hdr_len;
 228         src_len -= na->up.virt_hdr_len;
 229         dst += dst_na->up.virt_hdr_len;
 230         dst_len = dst_na->up.virt_hdr_len + src_len;
 231
 232         /* Here it could be dst_len == 0 (which implies src_len == 0),
 233          * so we avoid passing a zero length fragment.
 234          */
 235         if (dst_len == 0) {
 236                 ft_p++;
 237                 src = ft_p->ft_buf;
 238                 src_len = ft_p->ft_len;
 239                 dst_len = src_len;
 240         }
 241
 242         if (vh && vh->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
 243                 u_int gso_bytes = 0;
 244                 /* Length of the GSO packet header. */
 245                 u_int gso_hdr_len = 0;
 246                 /* Pointer to the GSO packet header. Assume it is in a single fragment. */
 247                 uint8_t *gso_hdr = NULL;
 248                 /* Index of the current segment. */
 249                 u_int gso_idx = 0;
 250                 /* Payload data bytes segmented so far (e.g. TCP data bytes). */
 251                 u_int segmented_bytes = 0;
 252                 /* Is this an IPv4 or IPv6 GSO packet? */
 253                 u_int ipv4 = 0;
 254                 /* Length of the IP header (20 if IPv4, 40 if IPv6). */
 255                 u_int iphlen = 0;
 256                 /* Length of the Ethernet header (18 if 802.1q, otherwise 14). */
 257                 u_int ethhlen = 14;
 258                 /* Is this a TCP or an UDP GSO packet? */
 259                 u_int tcp = ((vh->gso_type & ~VIRTIO_NET_HDR_GSO_ECN)
 260                                 == VIRTIO_NET_HDR_GSO_UDP) ? 0 : 1;
 261
 262                 /* Segment the GSO packet contained into the input slots (frags). */
 263                 for (;;) {
 264                         size_t copy;
 265
 266                         if (dst_slots >= *howmany) {
 267                                 /* We still have work to do, but we've run out of
 268                                  * dst slots, so we have to drop the packet. */
 269                                 nm_prdis(1, "Not enough slots, dropping GSO packet");
 270                                 return;
 271                         }
 272
 273                         /* Grab the GSO header if we don't have it. */
 274                         if (!gso_hdr) {
 275                                 uint16_t ethertype;
 276
 277                                 gso_hdr = src;
 278
 279                                 /* Look at the 'Ethertype' field to see if this packet
 280                                  * is IPv4 or IPv6, taking into account VLAN
 281                                  * encapsulation. */
 282                                 for (;;) {
 283                                         if (src_len < ethhlen) {
 284                                                 nm_prlim(1, "Short GSO fragment [eth], dropping");
 285                                                 return;
 286                                         }
 287                                         ethertype = be16toh(*((uint16_t *)
 288                                                             (gso_hdr + ethhlen - 2)));
 289                                         if (ethertype != 0x8100) /* not 802.1q */
 290                                                 break;
 291                                         ethhlen += 4;
 292                                 }
 293                                 switch (ethertype) {
 294                                         case 0x0800:  /* IPv4 */
 295                                         {
 296                                                 struct nm_iphdr *iph = (struct nm_iphdr *)
 297                                                                         (gso_hdr + ethhlen);
 298
 299                                                 if (src_len < ethhlen + 20) {
 300                                                         nm_prlim(1, "Short GSO fragment "
 301                                                               "[IPv4], dropping");
 302                                                         return;
 303                                                 }
 304                                                 ipv4 = 1;
 305                                                 iphlen = 4 * (iph->version_ihl & 0x0F);
 306                                                 break;
 307                                         }
 308                                         case 0x86DD:  /* IPv6 */
 309                                                 ipv4 = 0;
 310                                                 iphlen = 40;
 311                                                 break;
 312                                         default:
 313                                                 nm_prlim(1, "Unsupported ethertype, "
 314                                                       "dropping GSO packet");
 315                                                 return;
 316                                 }
 317                                 nm_prdis(3, "type=%04x", ethertype);
 318
 319                                 if (src_len < ethhlen + iphlen) {
 320                                         nm_prlim(1, "Short GSO fragment [IP], dropping");
 321                                         return;
 322                                 }
 323
 324                                 /* Compute gso_hdr_len. For TCP we need to read the
 325                                  * content of the 'Data Offset' field.
 326                                  */
 327                                 if (tcp) {
 328                                         struct nm_tcphdr *tcph = (struct nm_tcphdr *)
 329                                                                 (gso_hdr + ethhlen + iphlen);
 330
 331                                         if (src_len < ethhlen + iphlen + 20) {
 332                                                 nm_prlim(1, "Short GSO fragment "
 333                                                                 "[TCP], dropping");
 334                                                 return;
 335                                         }
 336                                         gso_hdr_len = ethhlen + iphlen +
 337                                                       4 * (tcph->doff >> 4);
 338                                 } else {
 339                                         gso_hdr_len = ethhlen + iphlen + 8; /* UDP */
 340                                 }
 341
 342                                 if (src_len < gso_hdr_len) {
 343                                         nm_prlim(1, "Short GSO fragment [TCP/UDP], dropping");
 344                                         return;
 345                                 }
 346
 347                                 nm_prdis(3, "gso_hdr_len %u gso_mtu %d", gso_hdr_len,
 348                                                                    dst_na->mfs);
 349
 350                                 /* Advance source pointers. */
 351                                 src += gso_hdr_len;
 352                                 src_len -= gso_hdr_len;
 353                                 if (src_len == 0) {
 354                                         ft_p++;
 355                                         if (ft_p == ft_end)
 356                                                 break;
 357                                         src = ft_p->ft_buf;
 358                                         src_len = ft_p->ft_len;
 359                                 }
 360                         }
 361
 362                         /* Fill in the header of the current segment. */
 363                         if (gso_bytes == 0) {
 364                                 memcpy(dst, gso_hdr, gso_hdr_len);
 365                                 gso_bytes = gso_hdr_len;
 366                         }
 367
 368                         /* Fill in data and update source and dest pointers. */
 369                         copy = src_len;
 370                         if (gso_bytes + copy > dst_na->mfs)
 371                                 copy = dst_na->mfs - gso_bytes;
 372                         memcpy(dst + gso_bytes, src, copy);
 373                         gso_bytes += copy;
 374                         src += copy;
 375                         src_len -= copy;
 376
 377                         /* A segment is complete or we have processed all the
 378                            the GSO payload bytes. */
 379                         if (gso_bytes >= dst_na->mfs ||
 380                                 (src_len == 0 && ft_p + 1 == ft_end)) {
 381                                 /* After raw segmentation, we must fix some header
 382                                  * fields and compute checksums, in a protocol dependent
 383                                  * way. */
 384                                 gso_fix_segment(dst + ethhlen, gso_bytes - ethhlen,
 385                                                 ipv4, iphlen, tcp,
 386                                                 gso_idx, segmented_bytes,
 387                                                 src_len == 0 && ft_p + 1 == ft_end);
 388
 389                                 nm_prdis("frame %u completed with %d bytes", gso_idx, (int)gso_bytes);
 390                                 dst_slot->len = gso_bytes;
 391                                 dst_slot->flags = 0;
 392                                 dst_slots++;
 393                                 segmented_bytes += gso_bytes - gso_hdr_len;
 394
 395                                 gso_bytes = 0;
 396                                 gso_idx++;
 397
 398                                 /* Next destination slot. */
 399                                 j_cur = nm_next(j_cur, lim);
 400                                 dst_slot = &dst_ring->slot[j_cur];
 401                                 dst = NMB(&dst_na->up, dst_slot);
 402                         }
 403
 404                         /* Next input slot. */
 405                         if (src_len == 0) {
 406                                 ft_p++;
 407                                 if (ft_p == ft_end)
 408                                         break;
 409                                 src = ft_p->ft_buf;
 410                                 src_len = ft_p->ft_len;
 411                         }
 412                 }
 413                 nm_prdis(3, "%d bytes segmented", segmented_bytes);
 414
 415         } else {
 416                 /* Address of a checksum field into a destination slot. */
 417                 uint16_t *check = NULL;
 418                 /* Accumulator for an unfolded checksum. */
 419                 rawsum_t csum = 0;
 420
 421                 /* Process a non-GSO packet. */
 422
 423                 /* Init 'check' if necessary. */
 424                 if (vh && (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) {
 425                         if (unlikely(vh->csum_offset + vh->csum_start > src_len))
 426                                 nm_prerr("invalid checksum request");
 427                         else
 428                                 check = (uint16_t *)(dst + vh->csum_start +
 429                                                 vh->csum_offset);
 430                 }
 431
 432                 while (ft_p != ft_end) {
 433                         /* Init/update the packet checksum if needed. */
 434                         if (vh && (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) {
 435                                 if (!dst_slots)
 436                                         csum = nm_os_csum_raw(src + vh->csum_start,
 437                                                                 src_len - vh->csum_start, 0);
 438                                 else
 439                                         csum = nm_os_csum_raw(src, src_len, csum);
 440                         }
 441
 442                         /* Round to a multiple of 64 */
 443                         src_len = (src_len + 63) & ~63;
 444
 445                         if (ft_p->ft_flags & NS_INDIRECT) {
 446                                 if (copyin(src, dst, src_len)) {
 447                                         /* Invalid user pointer, pretend len is 0. */
 448                                         dst_len = 0;
 449                                 }
 450                         } else {
 451                                 memcpy(dst, src, (int)src_len);
 452                         }
 453                         dst_slot->len = dst_len;
 454                         dst_slots++;
 455
 456                         /* Next destination slot. */
 457                         j_cur = nm_next(j_cur, lim);
 458                         dst_slot = &dst_ring->slot[j_cur];
 459                         dst = NMB(&dst_na->up, dst_slot);
 460
 461                         /* Next source slot. */
 462                         ft_p++;
 463                         src = ft_p->ft_buf;
 464                         dst_len = src_len = ft_p->ft_len;
 465                 }
 466
 467                 /* Finalize (fold) the checksum if needed. */
 468                 if (check && vh && (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) {
 469                         *check = nm_os_csum_fold(csum);
 470                 }
 471                 nm_prdis(3, "using %u dst_slots", dst_slots);
 472
 473                 /* A second pass on the destination slots to set the slot flags,
 474                  * using the right number of destination slots.
 475                  */
 476                 while (j_start != j_cur) {
 477                         dst_slot = &dst_ring->slot[j_start];
 478                         dst_slot->flags = (dst_slots << 8)| NS_MOREFRAG;
 479                         j_start = nm_next(j_start, lim);
 480                 }
 481                 /* Clear NS_MOREFRAG flag on last entry. */
 482                 dst_slot->flags = (dst_slots << 8);
 483         }
 484
 485         /* Update howmany and j. This is to commit the use of
 486          * those slots in the destination ring. */
 487         if (unlikely(dst_slots > *howmany)) {
 488                 nm_prerr("bug: slot allocation error");
 489         }
 490         *j = j_cur;
 491         *howmany -= dst_slots;
 492 }