sys/dev/netmap/if_ptnet.c

   1 /*-
   2  * Copyright (c) 2016, Vincenzo Maffione
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 /* Driver for ptnet paravirtualized network device. */
  30
  31 #include <sys/cdefs.h>
  32
  33 #include <sys/types.h>
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/sockio.h>
  38 #include <sys/mbuf.h>
  39 #include <sys/malloc.h>
  40 #include <sys/module.h>
  41 #include <sys/socket.h>
  42 #include <sys/sysctl.h>
  43 #include <sys/lock.h>
  44 #include <sys/mutex.h>
  45 #include <sys/taskqueue.h>
  46 #include <sys/smp.h>
  47 #include <sys/time.h>
  48 #include <machine/smp.h>
  49
  50 #include <vm/uma.h>
  51 #include <vm/vm.h>
  52 #include <vm/pmap.h>
  53
  54 #include <net/ethernet.h>
  55 #include <net/if.h>
  56 #include <net/if_var.h>
  57 #include <net/if_arp.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_types.h>
  60 #include <net/if_media.h>
  61 #include <net/if_vlan_var.h>
  62 #include <net/bpf.h>
  63
  64 #include <netinet/in_systm.h>
  65 #include <netinet/in.h>
  66 #include <netinet/ip.h>
  67 #include <netinet/ip6.h>
  68 #include <netinet6/ip6_var.h>
  69 #include <netinet/udp.h>
  70 #include <netinet/tcp.h>
  71
  72 #include <machine/bus.h>
  73 #include <machine/resource.h>
  74 #include <sys/bus.h>
  75 #include <sys/rman.h>
  76
  77 #include <dev/pci/pcivar.h>
  78 #include <dev/pci/pcireg.h>
  79
  80 #include "opt_inet.h"
  81 #include "opt_inet6.h"
  82
  83 #include <sys/selinfo.h>
  84 #include <net/netmap.h>
  85 #include <dev/netmap/netmap_kern.h>
  86 #include <net/netmap_virt.h>
  87 #include <dev/netmap/netmap_mem2.h>
  88 #include <dev/virtio/network/virtio_net.h>
  89
  90 #ifndef INET
  91 #error "INET not defined, cannot support offloadings"
  92 #endif
  93
  94 #if __FreeBSD_version >= 1100000
  95 static uint64_t ptnet_get_counter(if_t, ift_counter);
  96 #else
  97 typedef struct ifnet *if_t;
  98 #define if_getsoftc(_ifp)   (_ifp)->if_softc
  99 #endif
 100
 101 //#define PTNETMAP_STATS
 102 //#define DEBUG
 103 #ifdef DEBUG
 104 #define DBG(x) x
 105 #else   /* !DEBUG */
 106 #define DBG(x)
 107 #endif  /* !DEBUG */
 108
 109 extern int ptnet_vnet_hdr; /* Tunable parameter */
 110
 111 struct ptnet_softc;
 112
 113 struct ptnet_queue_stats {
 114         uint64_t        packets; /* if_[io]packets */
 115         uint64_t        bytes;   /* if_[io]bytes */
 116         uint64_t        errors;  /* if_[io]errors */
 117         uint64_t        iqdrops; /* if_iqdrops */
 118         uint64_t        mcasts;  /* if_[io]mcasts */
 119 #ifdef PTNETMAP_STATS
 120         uint64_t        intrs;
 121         uint64_t        kicks;
 122 #endif /* PTNETMAP_STATS */
 123 };
 124
 125 struct ptnet_queue {
 126         struct ptnet_softc              *sc;
 127         struct                          resource *irq;
 128         void                            *cookie;
 129         int                             kring_id;
 130         struct nm_csb_atok              *atok;
 131         struct nm_csb_ktoa              *ktoa;
 132         unsigned int                    kick;
 133         struct mtx                      lock;
 134         struct buf_ring                 *bufring; /* for TX queues */
 135         struct ptnet_queue_stats        stats;
 136 #ifdef PTNETMAP_STATS
 137         struct ptnet_queue_stats        last_stats;
 138 #endif /* PTNETMAP_STATS */
 139         struct taskqueue                *taskq;
 140         struct task                     task;
 141         char                            lock_name[16];
 142 };
 143
 144 #define PTNET_Q_LOCK(_pq)       mtx_lock(&(_pq)->lock)
 145 #define PTNET_Q_TRYLOCK(_pq)    mtx_trylock(&(_pq)->lock)
 146 #define PTNET_Q_UNLOCK(_pq)     mtx_unlock(&(_pq)->lock)
 147
 148 struct ptnet_softc {
 149         device_t                dev;
 150         if_t                    ifp;
 151         struct ifmedia          media;
 152         struct mtx              lock;
 153         char                    lock_name[16];
 154         char                    hwaddr[ETHER_ADDR_LEN];
 155
 156         /* Mirror of PTFEAT register. */
 157         uint32_t                ptfeatures;
 158         unsigned int            vnet_hdr_len;
 159
 160         /* PCI BARs support. */
 161         struct resource         *iomem;
 162         struct resource         *msix_mem;
 163
 164         unsigned int            num_rings;
 165         unsigned int            num_tx_rings;
 166         struct ptnet_queue      *queues;
 167         struct ptnet_queue      *rxqueues;
 168         struct nm_csb_atok      *csb_gh;
 169         struct nm_csb_ktoa      *csb_hg;
 170
 171         unsigned int            min_tx_space;
 172
 173         struct netmap_pt_guest_adapter *ptna;
 174
 175         struct callout          tick;
 176 #ifdef PTNETMAP_STATS
 177         struct timeval          last_ts;
 178 #endif /* PTNETMAP_STATS */
 179 };
 180
 181 #define PTNET_CORE_LOCK(_sc)    mtx_lock(&(_sc)->lock)
 182 #define PTNET_CORE_UNLOCK(_sc)  mtx_unlock(&(_sc)->lock)
 183
 184 static int      ptnet_probe(device_t);
 185 static int      ptnet_attach(device_t);
 186 static int      ptnet_detach(device_t);
 187 static int      ptnet_suspend(device_t);
 188 static int      ptnet_resume(device_t);
 189 static int      ptnet_shutdown(device_t);
 190
 191 static void     ptnet_init(void *opaque);
 192 static int      ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data);
 193 static int      ptnet_init_locked(struct ptnet_softc *sc);
 194 static int      ptnet_stop(struct ptnet_softc *sc);
 195 static int      ptnet_transmit(if_t ifp, struct mbuf *m);
 196 static int      ptnet_drain_transmit_queue(struct ptnet_queue *pq,
 197                                            unsigned int budget,
 198                                            bool may_resched);
 199 static void     ptnet_qflush(if_t ifp);
 200 static void     ptnet_tx_task(void *context, int pending);
 201
 202 static int      ptnet_media_change(if_t ifp);
 203 static void     ptnet_media_status(if_t ifp, struct ifmediareq *ifmr);
 204 #ifdef PTNETMAP_STATS
 205 static void     ptnet_tick(void *opaque);
 206 #endif
 207
 208 static int      ptnet_irqs_init(struct ptnet_softc *sc);
 209 static void     ptnet_irqs_fini(struct ptnet_softc *sc);
 210
 211 static uint32_t ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd);
 212 static int      ptnet_nm_config(struct netmap_adapter *na,
 213                                 struct nm_config_info *info);
 214 static void     ptnet_update_vnet_hdr(struct ptnet_softc *sc);
 215 static int      ptnet_nm_register(struct netmap_adapter *na, int onoff);
 216 static int      ptnet_nm_txsync(struct netmap_kring *kring, int flags);
 217 static int      ptnet_nm_rxsync(struct netmap_kring *kring, int flags);
 218 static void     ptnet_nm_intr(struct netmap_adapter *na, int onoff);
 219
 220 static void     ptnet_tx_intr(void *opaque);
 221 static void     ptnet_rx_intr(void *opaque);
 222
 223 static unsigned ptnet_rx_discard(struct netmap_kring *kring,
 224                                  unsigned int head);
 225 static int      ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget,
 226                              bool may_resched);
 227 static void     ptnet_rx_task(void *context, int pending);
 228
 229 #ifdef DEVICE_POLLING
 230 static poll_handler_t ptnet_poll;
 231 #endif
 232
 233 static device_method_t ptnet_methods[] = {
 234         DEVMETHOD(device_probe,                 ptnet_probe),
 235         DEVMETHOD(device_attach,                ptnet_attach),
 236         DEVMETHOD(device_detach,                ptnet_detach),
 237         DEVMETHOD(device_suspend,               ptnet_suspend),
 238         DEVMETHOD(device_resume,                ptnet_resume),
 239         DEVMETHOD(device_shutdown,              ptnet_shutdown),
 240         DEVMETHOD_END
 241 };
 242
 243 static driver_t ptnet_driver = {
 244         "ptnet",
 245         ptnet_methods,
 246         sizeof(struct ptnet_softc)
 247 };
 248
 249 /* We use (SI_ORDER_MIDDLE+2) here, see DEV_MODULE_ORDERED() invocation. */
 250 static devclass_t ptnet_devclass;
 251 DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, ptnet_devclass,
 252                       NULL, NULL, SI_ORDER_MIDDLE + 2);
 253
 254 static int
 255 ptnet_probe(device_t dev)
 256 {
 257         if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID ||
 258                 pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) {
 259                 return (ENXIO);
 260         }
 261
 262         device_set_desc(dev, "ptnet network adapter");
 263
 264         return (BUS_PROBE_DEFAULT);
 265 }
 266
 267 static inline void ptnet_kick(struct ptnet_queue *pq)
 268 {
 269 #ifdef PTNETMAP_STATS
 270         pq->stats.kicks ++;
 271 #endif /* PTNETMAP_STATS */
 272         bus_write_4(pq->sc->iomem, pq->kick, 0);
 273 }
 274
 275 #define PTNET_BUF_RING_SIZE     4096
 276 #define PTNET_RX_BUDGET         512
 277 #define PTNET_RX_BATCH          1
 278 #define PTNET_TX_BUDGET         512
 279 #define PTNET_TX_BATCH          64
 280 #define PTNET_HDR_SIZE          sizeof(struct virtio_net_hdr_mrg_rxbuf)
 281 #define PTNET_MAX_PKT_SIZE      65536
 282
 283 #define PTNET_CSUM_OFFLOAD      (CSUM_TCP | CSUM_UDP)
 284 #define PTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6)
 285 #define PTNET_ALL_OFFLOAD       (CSUM_TSO | PTNET_CSUM_OFFLOAD |\
 286                                  PTNET_CSUM_OFFLOAD_IPV6)
 287
 288 static int
 289 ptnet_attach(device_t dev)
 290 {
 291         uint32_t ptfeatures = 0;
 292         unsigned int num_rx_rings, num_tx_rings;
 293         struct netmap_adapter na_arg;
 294         unsigned int nifp_offset;
 295         struct ptnet_softc *sc;
 296         if_t ifp;
 297         uint32_t macreg;
 298         int err, rid;
 299         int i;
 300
 301         sc = device_get_softc(dev);
 302         sc->dev = dev;
 303
 304         /* Setup PCI resources. */
 305         pci_enable_busmaster(dev);
 306
 307         rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR);
 308         sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
 309                                            RF_ACTIVE);
 310         if (sc->iomem == NULL) {
 311                 device_printf(dev, "Failed to map I/O BAR\n");
 312                 return (ENXIO);
 313         }
 314
 315         /* Negotiate features with the hypervisor. */
 316         if (ptnet_vnet_hdr) {
 317                 ptfeatures |= PTNETMAP_F_VNET_HDR;
 318         }
 319         bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */
 320         ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */
 321         sc->ptfeatures = ptfeatures;
 322
 323         num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
 324         num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
 325         sc->num_rings = num_tx_rings + num_rx_rings;
 326         sc->num_tx_rings = num_tx_rings;
 327
 328         if (sc->num_rings * sizeof(struct nm_csb_atok) > PAGE_SIZE) {
 329                 device_printf(dev, "CSB cannot handle that many rings (%u)\n",
 330                                 sc->num_rings);
 331                 err = ENOMEM;
 332                 goto err_path;
 333         }
 334
 335         /* Allocate CSB and carry out CSB allocation protocol. */
 336         sc->csb_gh = contigmalloc(2*PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO,
 337                                   (size_t)0, -1UL, PAGE_SIZE, 0);
 338         if (sc->csb_gh == NULL) {
 339                 device_printf(dev, "Failed to allocate CSB\n");
 340                 err = ENOMEM;
 341                 goto err_path;
 342         }
 343         sc->csb_hg = (struct nm_csb_ktoa *)(((char *)sc->csb_gh) + PAGE_SIZE);
 344
 345         {
 346                 /*
 347                  * We use uint64_t rather than vm_paddr_t since we
 348                  * need 64 bit addresses even on 32 bit platforms.
 349                  */
 350                 uint64_t paddr = vtophys(sc->csb_gh);
 351
 352                 /* CSB allocation protocol: write to BAH first, then
 353                  * to BAL (for both GH and HG sections). */
 354                 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH,
 355                                 (paddr >> 32) & 0xffffffff);
 356                 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL,
 357                                 paddr & 0xffffffff);
 358                 paddr = vtophys(sc->csb_hg);
 359                 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH,
 360                                 (paddr >> 32) & 0xffffffff);
 361                 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL,
 362                                 paddr & 0xffffffff);
 363         }
 364
 365         /* Allocate and initialize per-queue data structures. */
 366         sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings,
 367                             M_DEVBUF, M_NOWAIT | M_ZERO);
 368         if (sc->queues == NULL) {
 369                 err = ENOMEM;
 370                 goto err_path;
 371         }
 372         sc->rxqueues = sc->queues + num_tx_rings;
 373
 374         for (i = 0; i < sc->num_rings; i++) {
 375                 struct ptnet_queue *pq = sc->queues + i;
 376
 377                 pq->sc = sc;
 378                 pq->kring_id = i;
 379                 pq->kick = PTNET_IO_KICK_BASE + 4 * i;
 380                 pq->atok = sc->csb_gh + i;
 381                 pq->ktoa = sc->csb_hg + i;
 382                 snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d",
 383                          device_get_nameunit(dev), i);
 384                 mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF);
 385                 if (i >= num_tx_rings) {
 386                         /* RX queue: fix kring_id. */
 387                         pq->kring_id -= num_tx_rings;
 388                 } else {
 389                         /* TX queue: allocate buf_ring. */
 390                         pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE,
 391                                                 M_DEVBUF, M_NOWAIT, &pq->lock);
 392                         if (pq->bufring == NULL) {
 393                                 err = ENOMEM;
 394                                 goto err_path;
 395                         }
 396                 }
 397         }
 398
 399         sc->min_tx_space = 64; /* Safe initial value. */
 400
 401         err = ptnet_irqs_init(sc);
 402         if (err) {
 403                 goto err_path;
 404         }
 405
 406         /* Setup Ethernet interface. */
 407         sc->ifp = ifp = if_alloc(IFT_ETHER);
 408         if (ifp == NULL) {
 409                 device_printf(dev, "Failed to allocate ifnet\n");
 410                 err = ENOMEM;
 411                 goto err_path;
 412         }
 413
 414         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 415         ifp->if_baudrate = IF_Gbps(10);
 416         ifp->if_softc = sc;
 417         ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
 418         ifp->if_init = ptnet_init;
 419         ifp->if_ioctl = ptnet_ioctl;
 420 #if __FreeBSD_version >= 1100000
 421         ifp->if_get_counter = ptnet_get_counter;
 422 #endif
 423         ifp->if_transmit = ptnet_transmit;
 424         ifp->if_qflush = ptnet_qflush;
 425
 426         ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change,
 427                      ptnet_media_status);
 428         ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);
 429         ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX);
 430
 431         macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI);
 432         sc->hwaddr[0] = (macreg >> 8) & 0xff;
 433         sc->hwaddr[1] = macreg & 0xff;
 434         macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO);
 435         sc->hwaddr[2] = (macreg >> 24) & 0xff;
 436         sc->hwaddr[3] = (macreg >> 16) & 0xff;
 437         sc->hwaddr[4] = (macreg >> 8) & 0xff;
 438         sc->hwaddr[5] = macreg & 0xff;
 439
 440         ether_ifattach(ifp, sc->hwaddr);
 441
 442         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
 443         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 444
 445         if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) {
 446                 /* Similarly to what the vtnet driver does, we can emulate
 447                  * VLAN offloadings by inserting and removing the 802.1Q
 448                  * header during transmit and receive. We are then able
 449                  * to do checksum offloading of VLAN frames. */
 450                 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6
 451                                         | IFCAP_VLAN_HWCSUM
 452                                         | IFCAP_TSO | IFCAP_LRO
 453                                         | IFCAP_VLAN_HWTSO
 454                                         | IFCAP_VLAN_HWTAGGING;
 455         }
 456
 457         ifp->if_capenable = ifp->if_capabilities;
 458 #ifdef DEVICE_POLLING
 459         /* Don't enable polling by default. */
 460         ifp->if_capabilities |= IFCAP_POLLING;
 461 #endif
 462         snprintf(sc->lock_name, sizeof(sc->lock_name),
 463                  "%s", device_get_nameunit(dev));
 464         mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF);
 465         callout_init_mtx(&sc->tick, &sc->lock, 0);
 466
 467         /* Prepare a netmap_adapter struct instance to do netmap_attach(). */
 468         nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS);
 469         memset(&na_arg, 0, sizeof(na_arg));
 470         na_arg.ifp = ifp;
 471         na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
 472         na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
 473         na_arg.num_tx_rings = num_tx_rings;
 474         na_arg.num_rx_rings = num_rx_rings;
 475         na_arg.nm_config = ptnet_nm_config;
 476         na_arg.nm_krings_create = ptnet_nm_krings_create;
 477         na_arg.nm_krings_delete = ptnet_nm_krings_delete;
 478         na_arg.nm_dtor = ptnet_nm_dtor;
 479         na_arg.nm_intr = ptnet_nm_intr;
 480         na_arg.nm_register = ptnet_nm_register;
 481         na_arg.nm_txsync = ptnet_nm_txsync;
 482         na_arg.nm_rxsync = ptnet_nm_rxsync;
 483
 484         netmap_pt_guest_attach(&na_arg, nifp_offset,
 485                                 bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID));
 486
 487         /* Now a netmap adapter for this ifp has been allocated, and it
 488          * can be accessed through NA(ifp). We also have to initialize the CSB
 489          * pointer. */
 490         sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp);
 491
 492         /* If virtio-net header was negotiated, set the virt_hdr_len field in
 493          * the netmap adapter, to inform users that this netmap adapter requires
 494          * the application to deal with the headers. */
 495         ptnet_update_vnet_hdr(sc);
 496
 497         device_printf(dev, "%s() completed\n", __func__);
 498
 499         return (0);
 500
 501 err_path:
 502         ptnet_detach(dev);
 503         return err;
 504 }
 505
 506 /* Stop host sync-kloop if it was running. */
 507 static void
 508 ptnet_device_shutdown(struct ptnet_softc *sc)
 509 {
 510         ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
 511         bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 0);
 512         bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 0);
 513         bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 0);
 514         bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 0);
 515 }
 516
 517 static int
 518 ptnet_detach(device_t dev)
 519 {
 520         struct ptnet_softc *sc = device_get_softc(dev);
 521         int i;
 522
 523         ptnet_device_shutdown(sc);
 524
 525 #ifdef DEVICE_POLLING
 526         if (sc->ifp->if_capenable & IFCAP_POLLING) {
 527                 ether_poll_deregister(sc->ifp);
 528         }
 529 #endif
 530         callout_drain(&sc->tick);
 531
 532         if (sc->queues) {
 533                 /* Drain taskqueues before calling if_detach. */
 534                 for (i = 0; i < sc->num_rings; i++) {
 535                         struct ptnet_queue *pq = sc->queues + i;
 536
 537                         if (pq->taskq) {
 538                                 taskqueue_drain(pq->taskq, &pq->task);
 539                         }
 540                 }
 541         }
 542
 543         if (sc->ifp) {
 544                 ether_ifdetach(sc->ifp);
 545
 546                 /* Uninitialize netmap adapters for this device. */
 547                 netmap_detach(sc->ifp);
 548
 549                 ifmedia_removeall(&sc->media);
 550                 if_free(sc->ifp);
 551                 sc->ifp = NULL;
 552         }
 553
 554         ptnet_irqs_fini(sc);
 555
 556         if (sc->csb_gh) {
 557                 contigfree(sc->csb_gh, 2*PAGE_SIZE, M_DEVBUF);
 558                 sc->csb_gh = NULL;
 559                 sc->csb_hg = NULL;
 560         }
 561
 562         if (sc->queues) {
 563                 for (i = 0; i < sc->num_rings; i++) {
 564                         struct ptnet_queue *pq = sc->queues + i;
 565
 566                         if (mtx_initialized(&pq->lock)) {
 567                                 mtx_destroy(&pq->lock);
 568                         }
 569                         if (pq->bufring != NULL) {
 570                                 buf_ring_free(pq->bufring, M_DEVBUF);
 571                         }
 572                 }
 573                 free(sc->queues, M_DEVBUF);
 574                 sc->queues = NULL;
 575         }
 576
 577         if (sc->iomem) {
 578                 bus_release_resource(dev, SYS_RES_IOPORT,
 579                                      PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem);
 580                 sc->iomem = NULL;
 581         }
 582
 583         mtx_destroy(&sc->lock);
 584
 585         device_printf(dev, "%s() completed\n", __func__);
 586
 587         return (0);
 588 }
 589
 590 static int
 591 ptnet_suspend(device_t dev)
 592 {
 593         struct ptnet_softc *sc = device_get_softc(dev);
 594
 595         (void)sc;
 596
 597         return (0);
 598 }
 599
 600 static int
 601 ptnet_resume(device_t dev)
 602 {
 603         struct ptnet_softc *sc = device_get_softc(dev);
 604
 605         (void)sc;
 606
 607         return (0);
 608 }
 609
 610 static int
 611 ptnet_shutdown(device_t dev)
 612 {
 613         struct ptnet_softc *sc = device_get_softc(dev);
 614
 615         ptnet_device_shutdown(sc);
 616
 617         return (0);
 618 }
 619
 620 static int
 621 ptnet_irqs_init(struct ptnet_softc *sc)
 622 {
 623         int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR);
 624         int nvecs = sc->num_rings;
 625         device_t dev = sc->dev;
 626         int err = ENOSPC;
 627         int cpu_cur;
 628         int i;
 629
 630         if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0)  {
 631                 device_printf(dev, "Could not find MSI-X capability\n");
 632                 return (ENXIO);
 633         }
 634
 635         sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 636                                               &rid, RF_ACTIVE);
 637         if (sc->msix_mem == NULL) {
 638                 device_printf(dev, "Failed to allocate MSIX PCI BAR\n");
 639                 return (ENXIO);
 640         }
 641
 642         if (pci_msix_count(dev) < nvecs) {
 643                 device_printf(dev, "Not enough MSI-X vectors\n");
 644                 goto err_path;
 645         }
 646
 647         err = pci_alloc_msix(dev, &nvecs);
 648         if (err) {
 649                 device_printf(dev, "Failed to allocate MSI-X vectors\n");
 650                 goto err_path;
 651         }
 652
 653         for (i = 0; i < nvecs; i++) {
 654                 struct ptnet_queue *pq = sc->queues + i;
 655
 656                 rid = i + 1;
 657                 pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 658                                                  RF_ACTIVE);
 659                 if (pq->irq == NULL) {
 660                         device_printf(dev, "Failed to allocate interrupt "
 661                                            "for queue #%d\n", i);
 662                         err = ENOSPC;
 663                         goto err_path;
 664                 }
 665         }
 666
 667         cpu_cur = CPU_FIRST();
 668         for (i = 0; i < nvecs; i++) {
 669                 struct ptnet_queue *pq = sc->queues + i;
 670                 void (*handler)(void *) = ptnet_tx_intr;
 671
 672                 if (i >= sc->num_tx_rings) {
 673                         handler = ptnet_rx_intr;
 674                 }
 675                 err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE,
 676                                      NULL /* intr_filter */, handler,
 677                                      pq, &pq->cookie);
 678                 if (err) {
 679                         device_printf(dev, "Failed to register intr handler "
 680                                            "for queue #%d\n", i);
 681                         goto err_path;
 682                 }
 683
 684                 bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i);
 685 #if 0
 686                 bus_bind_intr(sc->dev, pq->irq, cpu_cur);
 687 #endif
 688                 cpu_cur = CPU_NEXT(cpu_cur);
 689         }
 690
 691         device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs);
 692
 693         cpu_cur = CPU_FIRST();
 694         for (i = 0; i < nvecs; i++) {
 695                 struct ptnet_queue *pq = sc->queues + i;
 696                 static void (*handler)(void *context, int pending);
 697
 698                 handler = (i < sc->num_tx_rings) ? ptnet_tx_task : ptnet_rx_task;
 699
 700                 TASK_INIT(&pq->task, 0, handler, pq);
 701                 pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT,
 702                                         taskqueue_thread_enqueue, &pq->taskq);
 703                 taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d",
 704                                         device_get_nameunit(sc->dev), cpu_cur);
 705                 cpu_cur = CPU_NEXT(cpu_cur);
 706         }
 707
 708         return 0;
 709 err_path:
 710         ptnet_irqs_fini(sc);
 711         return err;
 712 }
 713
 714 static void
 715 ptnet_irqs_fini(struct ptnet_softc *sc)
 716 {
 717         device_t dev = sc->dev;
 718         int i;
 719
 720         for (i = 0; i < sc->num_rings; i++) {
 721                 struct ptnet_queue *pq = sc->queues + i;
 722
 723                 if (pq->taskq) {
 724                         taskqueue_free(pq->taskq);
 725                         pq->taskq = NULL;
 726                 }
 727
 728                 if (pq->cookie) {
 729                         bus_teardown_intr(dev, pq->irq, pq->cookie);
 730                         pq->cookie = NULL;
 731                 }
 732
 733                 if (pq->irq) {
 734                         bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq);
 735                         pq->irq = NULL;
 736                 }
 737         }
 738
 739         if (sc->msix_mem) {
 740                 pci_release_msi(dev);
 741
 742                 bus_release_resource(dev, SYS_RES_MEMORY,
 743                                      PCIR_BAR(PTNETMAP_MSIX_PCI_BAR),
 744                                      sc->msix_mem);
 745                 sc->msix_mem = NULL;
 746         }
 747 }
 748
 749 static void
 750 ptnet_init(void *opaque)
 751 {
 752         struct ptnet_softc *sc = opaque;
 753
 754         PTNET_CORE_LOCK(sc);
 755         ptnet_init_locked(sc);
 756         PTNET_CORE_UNLOCK(sc);
 757 }
 758
 759 static int
 760 ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data)
 761 {
 762         struct ptnet_softc *sc = if_getsoftc(ifp);
 763         device_t dev = sc->dev;
 764         struct ifreq *ifr = (struct ifreq *)data;
 765         int mask __unused, err = 0;
 766
 767         switch (cmd) {
 768         case SIOCSIFFLAGS:
 769                 device_printf(dev, "SIOCSIFFLAGS %x\n", ifp->if_flags);
 770                 PTNET_CORE_LOCK(sc);
 771                 if (ifp->if_flags & IFF_UP) {
 772                         /* Network stack wants the iff to be up. */
 773                         err = ptnet_init_locked(sc);
 774                 } else {
 775                         /* Network stack wants the iff to be down. */
 776                         err = ptnet_stop(sc);
 777                 }
 778                 /* We don't need to do nothing to support IFF_PROMISC,
 779                  * since that is managed by the backend port. */
 780                 PTNET_CORE_UNLOCK(sc);
 781                 break;
 782
 783         case SIOCSIFCAP:
 784                 device_printf(dev, "SIOCSIFCAP %x %x\n",
 785                               ifr->ifr_reqcap, ifp->if_capenable);
 786                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 787 #ifdef DEVICE_POLLING
 788                 if (mask & IFCAP_POLLING) {
 789                         struct ptnet_queue *pq;
 790                         int i;
 791
 792                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
 793                                 err = ether_poll_register(ptnet_poll, ifp);
 794                                 if (err) {
 795                                         break;
 796                                 }
 797                                 /* Stop queues and sync with taskqueues. */
 798                                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 799                                 for (i = 0; i < sc->num_rings; i++) {
 800                                         pq = sc-> queues + i;
 801                                         /* Make sure the worker sees the
 802                                          * IFF_DRV_RUNNING down. */
 803                                         PTNET_Q_LOCK(pq);
 804                                         pq->atok->appl_need_kick = 0;
 805                                         PTNET_Q_UNLOCK(pq);
 806                                         /* Wait for rescheduling to finish. */
 807                                         if (pq->taskq) {
 808                                                 taskqueue_drain(pq->taskq,
 809                                                                 &pq->task);
 810                                         }
 811                                 }
 812                                 ifp->if_drv_flags |= IFF_DRV_RUNNING;
 813                         } else {
 814                                 err = ether_poll_deregister(ifp);
 815                                 for (i = 0; i < sc->num_rings; i++) {
 816                                         pq = sc-> queues + i;
 817                                         PTNET_Q_LOCK(pq);
 818                                         pq->atok->appl_need_kick = 1;
 819                                         PTNET_Q_UNLOCK(pq);
 820                                 }
 821                         }
 822                 }
 823 #endif  /* DEVICE_POLLING */
 824                 ifp->if_capenable = ifr->ifr_reqcap;
 825                 break;
 826
 827         case SIOCSIFMTU:
 828                 /* We support any reasonable MTU. */
 829                 if (ifr->ifr_mtu < ETHERMIN ||
 830                                 ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) {
 831                         err = EINVAL;
 832                 } else {
 833                         PTNET_CORE_LOCK(sc);
 834                         ifp->if_mtu = ifr->ifr_mtu;
 835                         PTNET_CORE_UNLOCK(sc);
 836                 }
 837                 break;
 838
 839         case SIOCSIFMEDIA:
 840         case SIOCGIFMEDIA:
 841                 err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
 842                 break;
 843
 844         default:
 845                 err = ether_ioctl(ifp, cmd, data);
 846                 break;
 847         }
 848
 849         return err;
 850 }
 851
 852 static int
 853 ptnet_init_locked(struct ptnet_softc *sc)
 854 {
 855         if_t ifp = sc->ifp;
 856         struct netmap_adapter *na_dr = &sc->ptna->dr.up;
 857         struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
 858         unsigned int nm_buf_size;
 859         int ret;
 860
 861         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 862                 return 0; /* nothing to do */
 863         }
 864
 865         device_printf(sc->dev, "%s\n", __func__);
 866
 867         /* Translate offload capabilities according to if_capenable. */
 868         ifp->if_hwassist = 0;
 869         if (ifp->if_capenable & IFCAP_TXCSUM)
 870                 ifp->if_hwassist |= PTNET_CSUM_OFFLOAD;
 871         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 872                 ifp->if_hwassist |= PTNET_CSUM_OFFLOAD_IPV6;
 873         if (ifp->if_capenable & IFCAP_TSO4)
 874                 ifp->if_hwassist |= CSUM_IP_TSO;
 875         if (ifp->if_capenable & IFCAP_TSO6)
 876                 ifp->if_hwassist |= CSUM_IP6_TSO;
 877
 878         /*
 879          * Prepare the interface for netmap mode access.
 880          */
 881         netmap_update_config(na_dr);
 882
 883         ret = netmap_mem_finalize(na_dr->nm_mem, na_dr);
 884         if (ret) {
 885                 device_printf(sc->dev, "netmap_mem_finalize() failed\n");
 886                 return ret;
 887         }
 888
 889         if (sc->ptna->backend_users == 0) {
 890                 ret = ptnet_nm_krings_create(na_nm);
 891                 if (ret) {
 892                         device_printf(sc->dev, "ptnet_nm_krings_create() "
 893                                                "failed\n");
 894                         goto err_mem_finalize;
 895                 }
 896
 897                 ret = netmap_mem_rings_create(na_dr);
 898                 if (ret) {
 899                         device_printf(sc->dev, "netmap_mem_rings_create() "
 900                                                "failed\n");
 901                         goto err_rings_create;
 902                 }
 903
 904                 ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut);
 905                 if (ret) {
 906                         device_printf(sc->dev, "netmap_mem_get_lut() "
 907                                                "failed\n");
 908                         goto err_get_lut;
 909                 }
 910         }
 911
 912         ret = ptnet_nm_register(na_dr, 1 /* on */);
 913         if (ret) {
 914                 goto err_register;
 915         }
 916
 917         nm_buf_size = NETMAP_BUF_SIZE(na_dr);
 918
 919         KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size"));
 920         sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2;
 921         device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__,
 922                       sc->min_tx_space);
 923 #ifdef PTNETMAP_STATS
 924         callout_reset(&sc->tick, hz, ptnet_tick, sc);
 925 #endif
 926
 927         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 928
 929         return 0;
 930
 931 err_register:
 932         memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut));
 933 err_get_lut:
 934         netmap_mem_rings_delete(na_dr);
 935 err_rings_create:
 936         ptnet_nm_krings_delete(na_nm);
 937 err_mem_finalize:
 938         netmap_mem_deref(na_dr->nm_mem, na_dr);
 939
 940         return ret;
 941 }
 942
 943 /* To be called under core lock. */
 944 static int
 945 ptnet_stop(struct ptnet_softc *sc)
 946 {
 947         if_t ifp = sc->ifp;
 948         struct netmap_adapter *na_dr = &sc->ptna->dr.up;
 949         struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
 950         int i;
 951
 952         device_printf(sc->dev, "%s\n", __func__);
 953
 954         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 955                 return 0; /* nothing to do */
 956         }
 957
 958         /* Clear the driver-ready flag, and synchronize with all the queues,
 959          * so that after this loop we are sure nobody is working anymore with
 960          * the device. This scheme is taken from the vtnet driver. */
 961         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 962         callout_stop(&sc->tick);
 963         for (i = 0; i < sc->num_rings; i++) {
 964                 PTNET_Q_LOCK(sc->queues + i);
 965                 PTNET_Q_UNLOCK(sc->queues + i);
 966         }
 967
 968         ptnet_nm_register(na_dr, 0 /* off */);
 969
 970         if (sc->ptna->backend_users == 0) {
 971                 netmap_mem_rings_delete(na_dr);
 972                 ptnet_nm_krings_delete(na_nm);
 973         }
 974         netmap_mem_deref(na_dr->nm_mem, na_dr);
 975
 976         return 0;
 977 }
 978
 979 static void
 980 ptnet_qflush(if_t ifp)
 981 {
 982         struct ptnet_softc *sc = if_getsoftc(ifp);
 983         int i;
 984
 985         /* Flush all the bufrings and do the interface flush. */
 986         for (i = 0; i < sc->num_rings; i++) {
 987                 struct ptnet_queue *pq = sc->queues + i;
 988                 struct mbuf *m;
 989
 990                 PTNET_Q_LOCK(pq);
 991                 if (pq->bufring) {
 992                         while ((m = buf_ring_dequeue_sc(pq->bufring))) {
 993                                 m_freem(m);
 994                         }
 995                 }
 996                 PTNET_Q_UNLOCK(pq);
 997         }
 998
 999         if_qflush(ifp);
1000 }
1001
1002 static int
1003 ptnet_media_change(if_t ifp)
1004 {
1005         struct ptnet_softc *sc = if_getsoftc(ifp);
1006         struct ifmedia *ifm = &sc->media;
1007
1008         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) {
1009                 return EINVAL;
1010         }
1011
1012         return 0;
1013 }
1014
1015 #if __FreeBSD_version >= 1100000
1016 static uint64_t
1017 ptnet_get_counter(if_t ifp, ift_counter cnt)
1018 {
1019         struct ptnet_softc *sc = if_getsoftc(ifp);
1020         struct ptnet_queue_stats stats[2];
1021         int i;
1022
1023         /* Accumulate statistics over the queues. */
1024         memset(stats, 0, sizeof(stats));
1025         for (i = 0; i < sc->num_rings; i++) {
1026                 struct ptnet_queue *pq = sc->queues + i;
1027                 int idx = (i < sc->num_tx_rings) ? 0 : 1;
1028
1029                 stats[idx].packets      += pq->stats.packets;
1030                 stats[idx].bytes        += pq->stats.bytes;
1031                 stats[idx].errors       += pq->stats.errors;
1032                 stats[idx].iqdrops      += pq->stats.iqdrops;
1033                 stats[idx].mcasts       += pq->stats.mcasts;
1034         }
1035
1036         switch (cnt) {
1037         case IFCOUNTER_IPACKETS:
1038                 return (stats[1].packets);
1039         case IFCOUNTER_IQDROPS:
1040                 return (stats[1].iqdrops);
1041         case IFCOUNTER_IERRORS:
1042                 return (stats[1].errors);
1043         case IFCOUNTER_OPACKETS:
1044                 return (stats[0].packets);
1045         case IFCOUNTER_OBYTES:
1046                 return (stats[0].bytes);
1047         case IFCOUNTER_OMCASTS:
1048                 return (stats[0].mcasts);
1049         default:
1050                 return (if_get_counter_default(ifp, cnt));
1051         }
1052 }
1053 #endif
1054
1055
1056 #ifdef PTNETMAP_STATS
1057 /* Called under core lock. */
1058 static void
1059 ptnet_tick(void *opaque)
1060 {
1061         struct ptnet_softc *sc = opaque;
1062         int i;
1063
1064         for (i = 0; i < sc->num_rings; i++) {
1065                 struct ptnet_queue *pq = sc->queues + i;
1066                 struct ptnet_queue_stats cur = pq->stats;
1067                 struct timeval now;
1068                 unsigned int delta;
1069
1070                 microtime(&now);
1071                 delta = now.tv_usec - sc->last_ts.tv_usec +
1072                         (now.tv_sec - sc->last_ts.tv_sec) * 1000000;
1073                 delta /= 1000; /* in milliseconds */
1074
1075                 if (delta == 0)
1076                         continue;
1077
1078                 device_printf(sc->dev, "#%d[%u ms]:pkts %lu, kicks %lu, "
1079                               "intr %lu\n", i, delta,
1080                               (cur.packets - pq->last_stats.packets),
1081                               (cur.kicks - pq->last_stats.kicks),
1082                               (cur.intrs - pq->last_stats.intrs));
1083                 pq->last_stats = cur;
1084         }
1085         microtime(&sc->last_ts);
1086         callout_schedule(&sc->tick, hz);
1087 }
1088 #endif /* PTNETMAP_STATS */
1089
1090 static void
1091 ptnet_media_status(if_t ifp, struct ifmediareq *ifmr)
1092 {
1093         /* We are always active, as the backend netmap port is
1094          * always open in netmap mode. */
1095         ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
1096         ifmr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
1097 }
1098
1099 static uint32_t
1100 ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd)
1101 {
1102         /*
1103          * Write a command and read back error status,
1104          * with zero meaning success.
1105          */
1106         bus_write_4(sc->iomem, PTNET_IO_PTCTL, cmd);
1107         return bus_read_4(sc->iomem, PTNET_IO_PTCTL);
1108 }
1109
1110 static int
1111 ptnet_nm_config(struct netmap_adapter *na, struct nm_config_info *info)
1112 {
1113         struct ptnet_softc *sc = if_getsoftc(na->ifp);
1114
1115         info->num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
1116         info->num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
1117         info->num_tx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
1118         info->num_rx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
1119         info->rx_buf_maxsize = NETMAP_BUF_SIZE(na);
1120
1121         device_printf(sc->dev, "txr %u, rxr %u, txd %u, rxd %u, rxbufsz %u\n",
1122                         info->num_tx_rings, info->num_rx_rings,
1123                         info->num_tx_descs, info->num_rx_descs,
1124                         info->rx_buf_maxsize);
1125
1126         return 0;
1127 }
1128
1129 static void
1130 ptnet_sync_from_csb(struct ptnet_softc *sc, struct netmap_adapter *na)
1131 {
1132         int i;
1133
1134         /* Sync krings from the host, reading from
1135          * CSB. */
1136         for (i = 0; i < sc->num_rings; i++) {
1137                 struct nm_csb_atok *atok = sc->queues[i].atok;
1138                 struct nm_csb_ktoa *ktoa = sc->queues[i].ktoa;
1139                 struct netmap_kring *kring;
1140
1141                 if (i < na->num_tx_rings) {
1142                         kring = na->tx_rings[i];
1143                 } else {
1144                         kring = na->rx_rings[i - na->num_tx_rings];
1145                 }
1146                 kring->rhead = kring->ring->head = atok->head;
1147                 kring->rcur = kring->ring->cur = atok->cur;
1148                 kring->nr_hwcur = ktoa->hwcur;
1149                 kring->nr_hwtail = kring->rtail =
1150                         kring->ring->tail = ktoa->hwtail;
1151
1152                 nm_prdis("%d,%d: csb {hc %u h %u c %u ht %u}", t, i,
1153                    ktoa->hwcur, atok->head, atok->cur,
1154                    ktoa->hwtail);
1155                 nm_prdis("%d,%d: kring {hc %u rh %u rc %u h %u c %u ht %u rt %u t %u}",
1156                    t, i, kring->nr_hwcur, kring->rhead, kring->rcur,
1157                    kring->ring->head, kring->ring->cur, kring->nr_hwtail,
1158                    kring->rtail, kring->ring->tail);
1159         }
1160 }
1161
1162 static void
1163 ptnet_update_vnet_hdr(struct ptnet_softc *sc)
1164 {
1165         unsigned int wanted_hdr_len = ptnet_vnet_hdr ? PTNET_HDR_SIZE : 0;
1166
1167         bus_write_4(sc->iomem, PTNET_IO_VNET_HDR_LEN, wanted_hdr_len);
1168         sc->vnet_hdr_len = bus_read_4(sc->iomem, PTNET_IO_VNET_HDR_LEN);
1169         sc->ptna->hwup.up.virt_hdr_len = sc->vnet_hdr_len;
1170 }
1171
1172 static int
1173 ptnet_nm_register(struct netmap_adapter *na, int onoff)
1174 {
1175         /* device-specific */
1176         if_t ifp = na->ifp;
1177         struct ptnet_softc *sc = if_getsoftc(ifp);
1178         int native = (na == &sc->ptna->hwup.up);
1179         struct ptnet_queue *pq;
1180         int ret = 0;
1181         int i;
1182
1183         if (!onoff) {
1184                 sc->ptna->backend_users--;
1185         }
1186
1187         /* If this is the last netmap client, guest interrupt enable flags may
1188          * be in arbitrary state. Since these flags are going to be used also
1189          * by the netdevice driver, we have to make sure to start with
1190          * notifications enabled. Also, schedule NAPI to flush pending packets
1191          * in the RX rings, since we will not receive further interrupts
1192          * until these will be processed. */
1193         if (native && !onoff && na->active_fds == 0) {
1194                 nm_prinf("Exit netmap mode, re-enable interrupts");
1195                 for (i = 0; i < sc->num_rings; i++) {
1196                         pq = sc->queues + i;
1197                         pq->atok->appl_need_kick = 1;
1198                 }
1199         }
1200
1201         if (onoff) {
1202                 if (sc->ptna->backend_users == 0) {
1203                         /* Initialize notification enable fields in the CSB. */
1204                         for (i = 0; i < sc->num_rings; i++) {
1205                                 pq = sc->queues + i;
1206                                 pq->ktoa->kern_need_kick = 1;
1207                                 pq->atok->appl_need_kick =
1208                                         (!(ifp->if_capenable & IFCAP_POLLING)
1209                                                 && i >= sc->num_tx_rings);
1210                         }
1211
1212                         /* Set the virtio-net header length. */
1213                         ptnet_update_vnet_hdr(sc);
1214
1215                         /* Make sure the host adapter passed through is ready
1216                          * for txsync/rxsync. */
1217                         ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_CREATE);
1218                         if (ret) {
1219                                 return ret;
1220                         }
1221
1222                         /* Align the guest krings and rings to the state stored
1223                          * in the CSB. */
1224                         ptnet_sync_from_csb(sc, na);
1225                 }
1226
1227                 /* If not native, don't call nm_set_native_flags, since we don't want
1228                  * to replace if_transmit method, nor set NAF_NETMAP_ON */
1229                 if (native) {
1230                         netmap_krings_mode_commit(na, onoff);
1231                         nm_set_native_flags(na);
1232                 }
1233
1234         } else {
1235                 if (native) {
1236                         nm_clear_native_flags(na);
1237                         netmap_krings_mode_commit(na, onoff);
1238                 }
1239
1240                 if (sc->ptna->backend_users == 0) {
1241                         ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
1242                 }
1243         }
1244
1245         if (onoff) {
1246                 sc->ptna->backend_users++;
1247         }
1248
1249         return ret;
1250 }
1251
1252 static int
1253 ptnet_nm_txsync(struct netmap_kring *kring, int flags)
1254 {
1255         struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1256         struct ptnet_queue *pq = sc->queues + kring->ring_id;
1257         bool notify;
1258
1259         notify = netmap_pt_guest_txsync(pq->atok, pq->ktoa, kring, flags);
1260         if (notify) {
1261                 ptnet_kick(pq);
1262         }
1263
1264         return 0;
1265 }
1266
1267 static int
1268 ptnet_nm_rxsync(struct netmap_kring *kring, int flags)
1269 {
1270         struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1271         struct ptnet_queue *pq = sc->rxqueues + kring->ring_id;
1272         bool notify;
1273
1274         notify = netmap_pt_guest_rxsync(pq->atok, pq->ktoa, kring, flags);
1275         if (notify) {
1276                 ptnet_kick(pq);
1277         }
1278
1279         return 0;
1280 }
1281
1282 static void
1283 ptnet_nm_intr(struct netmap_adapter *na, int onoff)
1284 {
1285         struct ptnet_softc *sc = if_getsoftc(na->ifp);
1286         int i;
1287
1288         for (i = 0; i < sc->num_rings; i++) {
1289                 struct ptnet_queue *pq = sc->queues + i;
1290                 pq->atok->appl_need_kick = onoff;
1291         }
1292 }
1293
1294 static void
1295 ptnet_tx_intr(void *opaque)
1296 {
1297         struct ptnet_queue *pq = opaque;
1298         struct ptnet_softc *sc = pq->sc;
1299
1300         DBG(device_printf(sc->dev, "Tx interrupt #%d\n", pq->kring_id));
1301 #ifdef PTNETMAP_STATS
1302         pq->stats.intrs ++;
1303 #endif /* PTNETMAP_STATS */
1304
1305         if (netmap_tx_irq(sc->ifp, pq->kring_id) != NM_IRQ_PASS) {
1306                 return;
1307         }
1308
1309         /* Schedule the tasqueue to flush process transmissions requests.
1310          * However, vtnet, if_em and if_igb just call ptnet_transmit() here,
1311          * at least when using MSI-X interrupts. The if_em driver, instead
1312          * schedule taskqueue when using legacy interrupts. */
1313         taskqueue_enqueue(pq->taskq, &pq->task);
1314 }
1315
1316 static void
1317 ptnet_rx_intr(void *opaque)
1318 {
1319         struct ptnet_queue *pq = opaque;
1320         struct ptnet_softc *sc = pq->sc;
1321         unsigned int unused;
1322
1323         DBG(device_printf(sc->dev, "Rx interrupt #%d\n", pq->kring_id));
1324 #ifdef PTNETMAP_STATS
1325         pq->stats.intrs ++;
1326 #endif /* PTNETMAP_STATS */
1327
1328         if (netmap_rx_irq(sc->ifp, pq->kring_id, &unused) != NM_IRQ_PASS) {
1329                 return;
1330         }
1331
1332         /* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts,
1333          * receive-side processing is executed directly in the interrupt
1334          * service routine. Alternatively, we may schedule the taskqueue. */
1335         ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
1336 }
1337
1338 /* The following offloadings-related functions are taken from the vtnet
1339  * driver, but the same functionality is required for the ptnet driver.
1340  * As a temporary solution, I copied this code from vtnet and I started
1341  * to generalize it (taking away driver-specific statistic accounting),
1342  * making as little modifications as possible.
1343  * In the future we need to share these functions between vtnet and ptnet.
1344  */
1345 static int
1346 ptnet_tx_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start)
1347 {
1348         struct ether_vlan_header *evh;
1349         int offset;
1350
1351         evh = mtod(m, struct ether_vlan_header *);
1352         if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1353                 /* BMV: We should handle nested VLAN tags too. */
1354                 *etype = ntohs(evh->evl_proto);
1355                 offset = sizeof(struct ether_vlan_header);
1356         } else {
1357                 *etype = ntohs(evh->evl_encap_proto);
1358                 offset = sizeof(struct ether_header);
1359         }
1360
1361         switch (*etype) {
1362 #if defined(INET)
1363         case ETHERTYPE_IP: {
1364                 struct ip *ip, iphdr;
1365                 if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
1366                         m_copydata(m, offset, sizeof(struct ip),
1367                             (caddr_t) &iphdr);
1368                         ip = &iphdr;
1369                 } else
1370                         ip = (struct ip *)(m->m_data + offset);
1371                 *proto = ip->ip_p;
1372                 *start = offset + (ip->ip_hl << 2);
1373                 break;
1374         }
1375 #endif
1376 #if defined(INET6)
1377         case ETHERTYPE_IPV6:
1378                 *proto = -1;
1379                 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
1380                 /* Assert the network stack sent us a valid packet. */
1381                 KASSERT(*start > offset,
1382                     ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
1383                     *start, offset, *proto));
1384                 break;
1385 #endif
1386         default:
1387                 /* Here we should increment the tx_csum_bad_ethtype counter. */
1388                 return (EINVAL);
1389         }
1390
1391         return (0);
1392 }
1393
1394 static int
1395 ptnet_tx_offload_tso(if_t ifp, struct mbuf *m, int eth_type,
1396                      int offset, bool allow_ecn, struct virtio_net_hdr *hdr)
1397 {
1398         static struct timeval lastecn;
1399         static int curecn;
1400         struct tcphdr *tcp, tcphdr;
1401
1402         if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) {
1403                 m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr);
1404                 tcp = &tcphdr;
1405         } else
1406                 tcp = (struct tcphdr *)(m->m_data + offset);
1407
1408         hdr->hdr_len = offset + (tcp->th_off << 2);
1409         hdr->gso_size = m->m_pkthdr.tso_segsz;
1410         hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
1411             VIRTIO_NET_HDR_GSO_TCPV6;
1412
1413         if (tcp->th_flags & TH_CWR) {
1414                 /*
1415                  * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD,
1416                  * ECN support is not on a per-interface basis, but globally via
1417                  * the net.inet.tcp.ecn.enable sysctl knob. The default is off.
1418                  */
1419                 if (!allow_ecn) {
1420                         if (ppsratecheck(&lastecn, &curecn, 1))
1421                                 if_printf(ifp,
1422                                     "TSO with ECN not negotiated with host\n");
1423                         return (ENOTSUP);
1424                 }
1425                 hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1426         }
1427
1428         /* Here we should increment tx_tso counter. */
1429
1430         return (0);
1431 }
1432
1433 static struct mbuf *
1434 ptnet_tx_offload(if_t ifp, struct mbuf *m, bool allow_ecn,
1435                  struct virtio_net_hdr *hdr)
1436 {
1437         int flags, etype, csum_start, proto, error;
1438
1439         flags = m->m_pkthdr.csum_flags;
1440
1441         error = ptnet_tx_offload_ctx(m, &etype, &proto, &csum_start);
1442         if (error)
1443                 goto drop;
1444
1445         if ((etype == ETHERTYPE_IP && flags & PTNET_CSUM_OFFLOAD) ||
1446             (etype == ETHERTYPE_IPV6 && flags & PTNET_CSUM_OFFLOAD_IPV6)) {
1447                 /*
1448                  * We could compare the IP protocol vs the CSUM_ flag too,
1449                  * but that really should not be necessary.
1450                  */
1451                 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1452                 hdr->csum_start = csum_start;
1453                 hdr->csum_offset = m->m_pkthdr.csum_data;
1454                 /* Here we should increment the tx_csum counter. */
1455         }
1456
1457         if (flags & CSUM_TSO) {
1458                 if (__predict_false(proto != IPPROTO_TCP)) {
1459                         /* Likely failed to correctly parse the mbuf.
1460                          * Here we should increment the tx_tso_not_tcp
1461                          * counter. */
1462                         goto drop;
1463                 }
1464
1465                 KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM,
1466                     ("%s: mbuf %p TSO without checksum offload %#x",
1467                     __func__, m, flags));
1468
1469                 error = ptnet_tx_offload_tso(ifp, m, etype, csum_start,
1470                                              allow_ecn, hdr);
1471                 if (error)
1472                         goto drop;
1473         }
1474
1475         return (m);
1476
1477 drop:
1478         m_freem(m);
1479         return (NULL);
1480 }
1481
1482 static void
1483 ptnet_vlan_tag_remove(struct mbuf *m)
1484 {
1485         struct ether_vlan_header *evh;
1486
1487         evh = mtod(m, struct ether_vlan_header *);
1488         m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
1489         m->m_flags |= M_VLANTAG;
1490
1491         /* Strip the 802.1Q header. */
1492         bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
1493             ETHER_HDR_LEN - ETHER_TYPE_LEN);
1494         m_adj(m, ETHER_VLAN_ENCAP_LEN);
1495 }
1496
1497 /*
1498  * Use the checksum offset in the VirtIO header to set the
1499  * correct CSUM_* flags.
1500  */
1501 static int
1502 ptnet_rx_csum_by_offset(struct mbuf *m, uint16_t eth_type, int ip_start,
1503                         struct virtio_net_hdr *hdr)
1504 {
1505 #if defined(INET) || defined(INET6)
1506         int offset = hdr->csum_start + hdr->csum_offset;
1507 #endif
1508
1509         /* Only do a basic sanity check on the offset. */
1510         switch (eth_type) {
1511 #if defined(INET)
1512         case ETHERTYPE_IP:
1513                 if (__predict_false(offset < ip_start + sizeof(struct ip)))
1514                         return (1);
1515                 break;
1516 #endif
1517 #if defined(INET6)
1518         case ETHERTYPE_IPV6:
1519                 if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr)))
1520                         return (1);
1521                 break;
1522 #endif
1523         default:
1524                 /* Here we should increment the rx_csum_bad_ethtype counter. */
1525                 return (1);
1526         }
1527
1528         /*
1529          * Use the offset to determine the appropriate CSUM_* flags. This is
1530          * a bit dirty, but we can get by with it since the checksum offsets
1531          * happen to be different. We assume the host host does not do IPv4
1532          * header checksum offloading.
1533          */
1534         switch (hdr->csum_offset) {
1535         case offsetof(struct udphdr, uh_sum):
1536         case offsetof(struct tcphdr, th_sum):
1537                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1538                 m->m_pkthdr.csum_data = 0xFFFF;
1539                 break;
1540         default:
1541                 /* Here we should increment the rx_csum_bad_offset counter. */
1542                 return (1);
1543         }
1544
1545         return (0);
1546 }
1547
1548 static int
1549 ptnet_rx_csum_by_parse(struct mbuf *m, uint16_t eth_type, int ip_start,
1550                        struct virtio_net_hdr *hdr)
1551 {
1552         int offset, proto;
1553
1554         switch (eth_type) {
1555 #if defined(INET)
1556         case ETHERTYPE_IP: {
1557                 struct ip *ip;
1558                 if (__predict_false(m->m_len < ip_start + sizeof(struct ip)))
1559                         return (1);
1560                 ip = (struct ip *)(m->m_data + ip_start);
1561                 proto = ip->ip_p;
1562                 offset = ip_start + (ip->ip_hl << 2);
1563                 break;
1564         }
1565 #endif
1566 #if defined(INET6)
1567         case ETHERTYPE_IPV6:
1568                 if (__predict_false(m->m_len < ip_start +
1569                     sizeof(struct ip6_hdr)))
1570                         return (1);
1571                 offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto);
1572                 if (__predict_false(offset < 0))
1573                         return (1);
1574                 break;
1575 #endif
1576         default:
1577                 /* Here we should increment the rx_csum_bad_ethtype counter. */
1578                 return (1);
1579         }
1580
1581         switch (proto) {
1582         case IPPROTO_TCP:
1583                 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr)))
1584                         return (1);
1585                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1586                 m->m_pkthdr.csum_data = 0xFFFF;
1587                 break;
1588         case IPPROTO_UDP:
1589                 if (__predict_false(m->m_len < offset + sizeof(struct udphdr)))
1590                         return (1);
1591                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1592                 m->m_pkthdr.csum_data = 0xFFFF;
1593                 break;
1594         default:
1595                 /*
1596                  * For the remaining protocols, FreeBSD does not support
1597                  * checksum offloading, so the checksum will be recomputed.
1598                  */
1599 #if 0
1600                 if_printf(ifp, "cksum offload of unsupported "
1601                     "protocol eth_type=%#x proto=%d csum_start=%d "
1602                     "csum_offset=%d\n", __func__, eth_type, proto,
1603                     hdr->csum_start, hdr->csum_offset);
1604 #endif
1605                 break;
1606         }
1607
1608         return (0);
1609 }
1610
1611 /*
1612  * Set the appropriate CSUM_* flags. Unfortunately, the information
1613  * provided is not directly useful to us. The VirtIO header gives the
1614  * offset of the checksum, which is all Linux needs, but this is not
1615  * how FreeBSD does things. We are forced to peek inside the packet
1616  * a bit.
1617  *
1618  * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD
1619  * could accept the offsets and let the stack figure it out.
1620  */
1621 static int
1622 ptnet_rx_csum(struct mbuf *m, struct virtio_net_hdr *hdr)
1623 {
1624         struct ether_header *eh;
1625         struct ether_vlan_header *evh;
1626         uint16_t eth_type;
1627         int offset, error;
1628
1629         eh = mtod(m, struct ether_header *);
1630         eth_type = ntohs(eh->ether_type);
1631         if (eth_type == ETHERTYPE_VLAN) {
1632                 /* BMV: We should handle nested VLAN tags too. */
1633                 evh = mtod(m, struct ether_vlan_header *);
1634                 eth_type = ntohs(evh->evl_proto);
1635                 offset = sizeof(struct ether_vlan_header);
1636         } else
1637                 offset = sizeof(struct ether_header);
1638
1639         if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1640                 error = ptnet_rx_csum_by_offset(m, eth_type, offset, hdr);
1641         else
1642                 error = ptnet_rx_csum_by_parse(m, eth_type, offset, hdr);
1643
1644         return (error);
1645 }
1646 /* End of offloading-related functions to be shared with vtnet. */
1647
1648 static void
1649 ptnet_ring_update(struct ptnet_queue *pq, struct netmap_kring *kring,
1650                   unsigned int head, unsigned int sync_flags)
1651 {
1652         struct netmap_ring *ring = kring->ring;
1653         struct nm_csb_atok *atok = pq->atok;
1654         struct nm_csb_ktoa *ktoa = pq->ktoa;
1655
1656         /* Some packets have been pushed to the netmap ring. We have
1657          * to tell the host to process the new packets, updating cur
1658          * and head in the CSB. */
1659         ring->head = ring->cur = head;
1660
1661         /* Mimic nm_txsync_prologue/nm_rxsync_prologue. */
1662         kring->rcur = kring->rhead = head;
1663
1664         nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
1665
1666         /* Kick the host if needed. */
1667         if (NM_ACCESS_ONCE(ktoa->kern_need_kick)) {
1668                 atok->sync_flags = sync_flags;
1669                 ptnet_kick(pq);
1670         }
1671 }
1672
1673 #define PTNET_TX_NOSPACE(_h, _k, _min)  \
1674         ((((_h) < (_k)->rtail) ? 0 : (_k)->nkr_num_slots) + \
1675                 (_k)->rtail - (_h)) < (_min)
1676
1677 /* This function may be called by the network stack, or by
1678  * by the taskqueue thread. */
1679 static int
1680 ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget,
1681                            bool may_resched)
1682 {
1683         struct ptnet_softc *sc = pq->sc;
1684         bool have_vnet_hdr = sc->vnet_hdr_len;
1685         struct netmap_adapter *na = &sc->ptna->dr.up;
1686         if_t ifp = sc->ifp;
1687         unsigned int batch_count = 0;
1688         struct nm_csb_atok *atok;
1689         struct nm_csb_ktoa *ktoa;
1690         struct netmap_kring *kring;
1691         struct netmap_ring *ring;
1692         struct netmap_slot *slot;
1693         unsigned int count = 0;
1694         unsigned int minspace;
1695         unsigned int head;
1696         unsigned int lim;
1697         struct mbuf *mhead;
1698         struct mbuf *mf;
1699         int nmbuf_bytes;
1700         uint8_t *nmbuf;
1701
1702         if (!PTNET_Q_TRYLOCK(pq)) {
1703                 /* We failed to acquire the lock, schedule the taskqueue. */
1704                 nm_prlim(1, "Deferring TX work");
1705                 if (may_resched) {
1706                         taskqueue_enqueue(pq->taskq, &pq->task);
1707                 }
1708
1709                 return 0;
1710         }
1711
1712         if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1713                 PTNET_Q_UNLOCK(pq);
1714                 nm_prlim(1, "Interface is down");
1715                 return ENETDOWN;
1716         }
1717
1718         atok = pq->atok;
1719         ktoa = pq->ktoa;
1720         kring = na->tx_rings[pq->kring_id];
1721         ring = kring->ring;
1722         lim = kring->nkr_num_slots - 1;
1723         head = ring->head;
1724         minspace = sc->min_tx_space;
1725
1726         while (count < budget) {
1727                 if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1728                         /* We ran out of slot, let's see if the host has
1729                          * freed up some, by reading hwcur and hwtail from
1730                          * the CSB. */
1731                         ptnet_sync_tail(ktoa, kring);
1732
1733                         if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1734                                 /* Still no slots available. Reactivate the
1735                                  * interrupts so that we can be notified
1736                                  * when some free slots are made available by
1737                                  * the host. */
1738                                 atok->appl_need_kick = 1;
1739
1740                                 /* Double check. We need a full barrier to
1741                                  * prevent the store to atok->appl_need_kick
1742                                  * to be reordered with the load from
1743                                  * ktoa->hwcur and ktoa->hwtail (store-load
1744                                  * barrier). */
1745                                 nm_stld_barrier();
1746                                 ptnet_sync_tail(ktoa, kring);
1747                                 if (likely(PTNET_TX_NOSPACE(head, kring,
1748                                                             minspace))) {
1749                                         break;
1750                                 }
1751
1752                                 nm_prlim(1, "Found more slots by doublecheck");
1753                                 /* More slots were freed before reactivating
1754                                  * the interrupts. */
1755                                 atok->appl_need_kick = 0;
1756                         }
1757                 }
1758
1759                 mhead = drbr_peek(ifp, pq->bufring);
1760                 if (!mhead) {
1761                         break;
1762                 }
1763
1764                 /* Initialize transmission state variables. */
1765                 slot = ring->slot + head;
1766                 nmbuf = NMB(na, slot);
1767                 nmbuf_bytes = 0;
1768
1769                 /* If needed, prepare the virtio-net header at the beginning
1770                  * of the first slot. */
1771                 if (have_vnet_hdr) {
1772                         struct virtio_net_hdr *vh =
1773                                         (struct virtio_net_hdr *)nmbuf;
1774
1775                         /* For performance, we could replace this memset() with
1776                          * two 8-bytes-wide writes. */
1777                         memset(nmbuf, 0, PTNET_HDR_SIZE);
1778                         if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) {
1779                                 mhead = ptnet_tx_offload(ifp, mhead, false,
1780                                                          vh);
1781                                 if (unlikely(!mhead)) {
1782                                         /* Packet dropped because errors
1783                                          * occurred while preparing the vnet
1784                                          * header. Let's go ahead with the next
1785                                          * packet. */
1786                                         pq->stats.errors ++;
1787                                         drbr_advance(ifp, pq->bufring);
1788                                         continue;
1789                                 }
1790                         }
1791                         nm_prdis(1, "%s: [csum_flags %lX] vnet hdr: flags %x "
1792                               "csum_start %u csum_ofs %u hdr_len = %u "
1793                               "gso_size %u gso_type %x", __func__,
1794                               mhead->m_pkthdr.csum_flags, vh->flags,
1795                               vh->csum_start, vh->csum_offset, vh->hdr_len,
1796                               vh->gso_size, vh->gso_type);
1797
1798                         nmbuf += PTNET_HDR_SIZE;
1799                         nmbuf_bytes += PTNET_HDR_SIZE;
1800                 }
1801
1802                 for (mf = mhead; mf; mf = mf->m_next) {
1803                         uint8_t *mdata = mf->m_data;
1804                         int mlen = mf->m_len;
1805
1806                         for (;;) {
1807                                 int copy = NETMAP_BUF_SIZE(na) - nmbuf_bytes;
1808
1809                                 if (mlen < copy) {
1810                                         copy = mlen;
1811                                 }
1812                                 memcpy(nmbuf, mdata, copy);
1813
1814                                 mdata += copy;
1815                                 mlen -= copy;
1816                                 nmbuf += copy;
1817                                 nmbuf_bytes += copy;
1818
1819                                 if (!mlen) {
1820                                         break;
1821                                 }
1822
1823                                 slot->len = nmbuf_bytes;
1824                                 slot->flags = NS_MOREFRAG;
1825
1826                                 head = nm_next(head, lim);
1827                                 KASSERT(head != ring->tail,
1828                                         ("Unexpectedly run out of TX space"));
1829                                 slot = ring->slot + head;
1830                                 nmbuf = NMB(na, slot);
1831                                 nmbuf_bytes = 0;
1832                         }
1833                 }
1834
1835                 /* Complete last slot and update head. */
1836                 slot->len = nmbuf_bytes;
1837                 slot->flags = 0;
1838                 head = nm_next(head, lim);
1839
1840                 /* Consume the packet just processed. */
1841                 drbr_advance(ifp, pq->bufring);
1842
1843                 /* Copy the packet to listeners. */
1844                 ETHER_BPF_MTAP(ifp, mhead);
1845
1846                 pq->stats.packets ++;
1847                 pq->stats.bytes += mhead->m_pkthdr.len;
1848                 if (mhead->m_flags & M_MCAST) {
1849                         pq->stats.mcasts ++;
1850                 }
1851
1852                 m_freem(mhead);
1853
1854                 count ++;
1855                 if (++batch_count == PTNET_TX_BATCH) {
1856                         ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1857                         batch_count = 0;
1858                 }
1859         }
1860
1861         if (batch_count) {
1862                 ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1863         }
1864
1865         if (count >= budget && may_resched) {
1866                 DBG(nm_prlim(1, "out of budget: resched, %d mbufs pending\n",
1867                                         drbr_inuse(ifp, pq->bufring)));
1868                 taskqueue_enqueue(pq->taskq, &pq->task);
1869         }
1870
1871         PTNET_Q_UNLOCK(pq);
1872
1873         return count;
1874 }
1875
1876 static int
1877 ptnet_transmit(if_t ifp, struct mbuf *m)
1878 {
1879         struct ptnet_softc *sc = if_getsoftc(ifp);
1880         struct ptnet_queue *pq;
1881         unsigned int queue_idx;
1882         int err;
1883
1884         DBG(device_printf(sc->dev, "transmit %p\n", m));
1885
1886         /* Insert 802.1Q header if needed. */
1887         if (m->m_flags & M_VLANTAG) {
1888                 m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1889                 if (m == NULL) {
1890                         return ENOBUFS;
1891                 }
1892                 m->m_flags &= ~M_VLANTAG;
1893         }
1894
1895         /* Get the flow-id if available. */
1896         queue_idx = (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) ?
1897                     m->m_pkthdr.flowid : curcpu;
1898
1899         if (unlikely(queue_idx >= sc->num_tx_rings)) {
1900                 queue_idx %= sc->num_tx_rings;
1901         }
1902
1903         pq = sc->queues + queue_idx;
1904
1905         err = drbr_enqueue(ifp, pq->bufring, m);
1906         if (err) {
1907                 /* ENOBUFS when the bufring is full */
1908                 nm_prlim(1, "%s: drbr_enqueue() failed %d\n",
1909                         __func__, err);
1910                 pq->stats.errors ++;
1911                 return err;
1912         }
1913
1914         if (ifp->if_capenable & IFCAP_POLLING) {
1915                 /* If polling is on, the transmit queues will be
1916                  * drained by the poller. */
1917                 return 0;
1918         }
1919
1920         err = ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
1921
1922         return (err < 0) ? err : 0;
1923 }
1924
1925 static unsigned int
1926 ptnet_rx_discard(struct netmap_kring *kring, unsigned int head)
1927 {
1928         struct netmap_ring *ring = kring->ring;
1929         struct netmap_slot *slot = ring->slot + head;
1930
1931         for (;;) {
1932                 head = nm_next(head, kring->nkr_num_slots - 1);
1933                 if (!(slot->flags & NS_MOREFRAG) || head == ring->tail) {
1934                         break;
1935                 }
1936                 slot = ring->slot + head;
1937         }
1938
1939         return head;
1940 }
1941
1942 static inline struct mbuf *
1943 ptnet_rx_slot(struct mbuf *mtail, uint8_t *nmbuf, unsigned int nmbuf_len)
1944 {
1945         uint8_t *mdata = mtod(mtail, uint8_t *) + mtail->m_len;
1946
1947         do {
1948                 unsigned int copy;
1949
1950                 if (mtail->m_len == MCLBYTES) {
1951                         struct mbuf *mf;
1952
1953                         mf = m_getcl(M_NOWAIT, MT_DATA, 0);
1954                         if (unlikely(!mf)) {
1955                                 return NULL;
1956                         }
1957
1958                         mtail->m_next = mf;
1959                         mtail = mf;
1960                         mdata = mtod(mtail, uint8_t *);
1961                         mtail->m_len = 0;
1962                 }
1963
1964                 copy = MCLBYTES - mtail->m_len;
1965                 if (nmbuf_len < copy) {
1966                         copy = nmbuf_len;
1967                 }
1968
1969                 memcpy(mdata, nmbuf, copy);
1970
1971                 nmbuf += copy;
1972                 nmbuf_len -= copy;
1973                 mdata += copy;
1974                 mtail->m_len += copy;
1975         } while (nmbuf_len);
1976
1977         return mtail;
1978 }
1979
1980 static int
1981 ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, bool may_resched)
1982 {
1983         struct ptnet_softc *sc = pq->sc;
1984         bool have_vnet_hdr = sc->vnet_hdr_len;
1985         struct nm_csb_atok *atok = pq->atok;
1986         struct nm_csb_ktoa *ktoa = pq->ktoa;
1987         struct netmap_adapter *na = &sc->ptna->dr.up;
1988         struct netmap_kring *kring = na->rx_rings[pq->kring_id];
1989         struct netmap_ring *ring = kring->ring;
1990         unsigned int const lim = kring->nkr_num_slots - 1;
1991         unsigned int batch_count = 0;
1992         if_t ifp = sc->ifp;
1993         unsigned int count = 0;
1994         uint32_t head;
1995
1996         PTNET_Q_LOCK(pq);
1997
1998         if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1999                 goto unlock;
2000         }
2001
2002         kring->nr_kflags &= ~NKR_PENDINTR;
2003
2004         head = ring->head;
2005         while (count < budget) {
2006                 uint32_t prev_head = head;
2007                 struct mbuf *mhead, *mtail;
2008                 struct virtio_net_hdr *vh;
2009                 struct netmap_slot *slot;
2010                 unsigned int nmbuf_len;
2011                 uint8_t *nmbuf;
2012                 int deliver = 1; /* the mbuf to the network stack. */
2013 host_sync:
2014                 if (head == ring->tail) {
2015                         /* We ran out of slot, let's see if the host has
2016                          * added some, by reading hwcur and hwtail from
2017                          * the CSB. */
2018                         ptnet_sync_tail(ktoa, kring);
2019
2020                         if (head == ring->tail) {
2021                                 /* Still no slots available. Reactivate
2022                                  * interrupts as they were disabled by the
2023                                  * host thread right before issuing the
2024                                  * last interrupt. */
2025                                 atok->appl_need_kick = 1;
2026
2027                                 /* Double check for more completed RX slots.
2028                                  * We need a full barrier to prevent the store
2029                                  * to atok->appl_need_kick to be reordered with
2030                                  * the load from ktoa->hwcur and ktoa->hwtail
2031                                  * (store-load barrier). */
2032                                 nm_stld_barrier();
2033                                 ptnet_sync_tail(ktoa, kring);
2034                                 if (likely(head == ring->tail)) {
2035                                         break;
2036                                 }
2037                                 atok->appl_need_kick = 0;
2038                         }
2039                 }
2040
2041                 /* Initialize ring state variables, possibly grabbing the
2042                  * virtio-net header. */
2043                 slot = ring->slot + head;
2044                 nmbuf = NMB(na, slot);
2045                 nmbuf_len = slot->len;
2046
2047                 vh = (struct virtio_net_hdr *)nmbuf;
2048                 if (have_vnet_hdr) {
2049                         if (unlikely(nmbuf_len < PTNET_HDR_SIZE)) {
2050                                 /* There is no good reason why host should
2051                                  * put the header in multiple netmap slots.
2052                                  * If this is the case, discard. */
2053                                 nm_prlim(1, "Fragmented vnet-hdr: dropping");
2054                                 head = ptnet_rx_discard(kring, head);
2055                                 pq->stats.iqdrops ++;
2056                                 deliver = 0;
2057                                 goto skip;
2058                         }
2059                         nm_prdis(1, "%s: vnet hdr: flags %x csum_start %u "
2060                               "csum_ofs %u hdr_len = %u gso_size %u "
2061                               "gso_type %x", __func__, vh->flags,
2062                               vh->csum_start, vh->csum_offset, vh->hdr_len,
2063                               vh->gso_size, vh->gso_type);
2064                         nmbuf += PTNET_HDR_SIZE;
2065                         nmbuf_len -= PTNET_HDR_SIZE;
2066                 }
2067
2068                 /* Allocate the head of a new mbuf chain.
2069                  * We use m_getcl() to allocate an mbuf with standard cluster
2070                  * size (MCLBYTES). In the future we could use m_getjcl()
2071                  * to choose different sizes. */
2072                 mhead = mtail = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
2073                 if (unlikely(mhead == NULL)) {
2074                         device_printf(sc->dev, "%s: failed to allocate mbuf "
2075                                       "head\n", __func__);
2076                         pq->stats.errors ++;
2077                         break;
2078                 }
2079
2080                 /* Initialize the mbuf state variables. */
2081                 mhead->m_pkthdr.len = nmbuf_len;
2082                 mtail->m_len = 0;
2083
2084                 /* Scan all the netmap slots containing the current packet. */
2085                 for (;;) {
2086                         DBG(device_printf(sc->dev, "%s: h %u t %u rcv frag "
2087                                           "len %u, flags %u\n", __func__,
2088                                           head, ring->tail, slot->len,
2089                                           slot->flags));
2090
2091                         mtail = ptnet_rx_slot(mtail, nmbuf, nmbuf_len);
2092                         if (unlikely(!mtail)) {
2093                                 /* Ouch. We ran out of memory while processing
2094                                  * a packet. We have to restore the previous
2095                                  * head position, free the mbuf chain, and
2096                                  * schedule the taskqueue to give the packet
2097                                  * another chance. */
2098                                 device_printf(sc->dev, "%s: failed to allocate"
2099                                         " mbuf frag, reset head %u --> %u\n",
2100                                         __func__, head, prev_head);
2101                                 head = prev_head;
2102                                 m_freem(mhead);
2103                                 pq->stats.errors ++;
2104                                 if (may_resched) {
2105                                         taskqueue_enqueue(pq->taskq,
2106                                                           &pq->task);
2107                                 }
2108                                 goto escape;
2109                         }
2110
2111                         /* We have to increment head irrespective of the
2112                          * NS_MOREFRAG being set or not. */
2113                         head = nm_next(head, lim);
2114
2115                         if (!(slot->flags & NS_MOREFRAG)) {
2116                                 break;
2117                         }
2118
2119                         if (unlikely(head == ring->tail)) {
2120                                 /* The very last slot prepared by the host has
2121                                  * the NS_MOREFRAG set. Drop it and continue
2122                                  * the outer cycle (to do the double-check). */
2123                                 nm_prlim(1, "Incomplete packet: dropping");
2124                                 m_freem(mhead);
2125                                 pq->stats.iqdrops ++;
2126                                 goto host_sync;
2127                         }
2128
2129                         slot = ring->slot + head;
2130                         nmbuf = NMB(na, slot);
2131                         nmbuf_len = slot->len;
2132                         mhead->m_pkthdr.len += nmbuf_len;
2133                 }
2134
2135                 mhead->m_pkthdr.rcvif = ifp;
2136                 mhead->m_pkthdr.csum_flags = 0;
2137
2138                 /* Store the queue idx in the packet header. */
2139                 mhead->m_pkthdr.flowid = pq->kring_id;
2140                 M_HASHTYPE_SET(mhead, M_HASHTYPE_OPAQUE);
2141
2142                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
2143                         struct ether_header *eh;
2144
2145                         eh = mtod(mhead, struct ether_header *);
2146                         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2147                                 ptnet_vlan_tag_remove(mhead);
2148                                 /*
2149                                  * With the 802.1Q header removed, update the
2150                                  * checksum starting location accordingly.
2151                                  */
2152                                 if (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
2153                                         vh->csum_start -= ETHER_VLAN_ENCAP_LEN;
2154                         }
2155                 }
2156
2157                 if (have_vnet_hdr && (vh->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM
2158                                         | VIRTIO_NET_HDR_F_DATA_VALID))) {
2159                         if (unlikely(ptnet_rx_csum(mhead, vh))) {
2160                                 m_freem(mhead);
2161                                 nm_prlim(1, "Csum offload error: dropping");
2162                                 pq->stats.iqdrops ++;
2163                                 deliver = 0;
2164                         }
2165                 }
2166
2167 skip:
2168                 count ++;
2169                 if (++batch_count >= PTNET_RX_BATCH) {
2170                         /* Some packets have been (or will be) pushed to the network
2171                          * stack. We need to update the CSB to tell the host about
2172                          * the new ring->cur and ring->head (RX buffer refill). */
2173                         ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
2174                         batch_count = 0;
2175                 }
2176
2177                 if (likely(deliver))  {
2178                         pq->stats.packets ++;
2179                         pq->stats.bytes += mhead->m_pkthdr.len;
2180
2181                         PTNET_Q_UNLOCK(pq);
2182                         (*ifp->if_input)(ifp, mhead);
2183                         PTNET_Q_LOCK(pq);
2184                         /* The ring->head index (and related indices) are
2185                          * updated under pq lock by ptnet_ring_update().
2186                          * Since we dropped the lock to call if_input(), we
2187                          * must reload ring->head and restart processing the
2188                          * ring from there. */
2189                         head = ring->head;
2190
2191                         if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
2192                                 /* The interface has gone down while we didn't
2193                                  * have the lock. Stop any processing and exit. */
2194                                 goto unlock;
2195                         }
2196                 }
2197         }
2198 escape:
2199         if (batch_count) {
2200                 ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
2201
2202         }
2203
2204         if (count >= budget && may_resched) {
2205                 /* If we ran out of budget or the double-check found new
2206                  * slots to process, schedule the taskqueue. */
2207                 DBG(nm_prlim(1, "out of budget: resched h %u t %u\n",
2208                                         head, ring->tail));
2209                 taskqueue_enqueue(pq->taskq, &pq->task);
2210         }
2211 unlock:
2212         PTNET_Q_UNLOCK(pq);
2213
2214         return count;
2215 }
2216
2217 static void
2218 ptnet_rx_task(void *context, int pending)
2219 {
2220         struct ptnet_queue *pq = context;
2221
2222         DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
2223         ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
2224 }
2225
2226 static void
2227 ptnet_tx_task(void *context, int pending)
2228 {
2229         struct ptnet_queue *pq = context;
2230
2231         DBG(nm_prlim(1, "%s: pq #%u\n", __func__, pq->kring_id));
2232         ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
2233 }
2234
2235 #ifdef DEVICE_POLLING
2236 /* We don't need to handle differently POLL_AND_CHECK_STATUS and
2237  * POLL_ONLY, since we don't have an Interrupt Status Register. */
2238 static int
2239 ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget)
2240 {
2241         struct ptnet_softc *sc = if_getsoftc(ifp);
2242         unsigned int queue_budget;
2243         unsigned int count = 0;
2244         bool borrow = false;
2245         int i;
2246
2247         KASSERT(sc->num_rings > 0, ("Found no queues in while polling ptnet"));
2248         queue_budget = MAX(budget / sc->num_rings, 1);
2249         nm_prlim(1, "Per-queue budget is %d", queue_budget);
2250
2251         while (budget) {
2252                 unsigned int rcnt = 0;
2253
2254                 for (i = 0; i < sc->num_rings; i++) {
2255                         struct ptnet_queue *pq = sc->queues + i;
2256
2257                         if (borrow) {
2258                                 queue_budget = MIN(queue_budget, budget);
2259                                 if (queue_budget == 0) {
2260                                         break;
2261                                 }
2262                         }
2263
2264                         if (i < sc->num_tx_rings) {
2265                                 rcnt += ptnet_drain_transmit_queue(pq,
2266                                                    queue_budget, false);
2267                         } else {
2268                                 rcnt += ptnet_rx_eof(pq, queue_budget,
2269                                                       false);
2270                         }
2271                 }
2272
2273                 if (!rcnt) {
2274                         /* A scan of the queues gave no result, we can
2275                          * stop here. */
2276                         break;
2277                 }
2278
2279                 if (rcnt > budget) {
2280                         /* This may happen when initial budget < sc->num_rings,
2281                          * since one packet budget is given to each queue
2282                          * anyway. Just pretend we didn't eat "so much". */
2283                         rcnt = budget;
2284                 }
2285                 count += rcnt;
2286                 budget -= rcnt;
2287                 borrow = true;
2288         }
2289
2290
2291         return count;
2292 }
2293 #endif /* DEVICE_POLLING */