sys/dev/netmap/if_ptnet.c

   1 /*-
   2  * Copyright (c) 2016, Vincenzo Maffione
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 /* Driver for ptnet paravirtualized network device. */
  30
  31 #include <sys/cdefs.h>
  32
  33 #include <sys/types.h>
  34 #include <sys/param.h>
  35 #include <sys/systm.h>
  36 #include <sys/kernel.h>
  37 #include <sys/sockio.h>
  38 #include <sys/mbuf.h>
  39 #include <sys/malloc.h>
  40 #include <sys/module.h>
  41 #include <sys/socket.h>
  42 #include <sys/sysctl.h>
  43 #include <sys/lock.h>
  44 #include <sys/mutex.h>
  45 #include <sys/taskqueue.h>
  46 #include <sys/smp.h>
  47 #include <sys/time.h>
  48 #include <machine/smp.h>
  49
  50 #include <vm/uma.h>
  51 #include <vm/vm.h>
  52 #include <vm/pmap.h>
  53
  54 #include <net/ethernet.h>
  55 #include <net/if.h>
  56 #include <net/if_var.h>
  57 #include <net/if_arp.h>
  58 #include <net/if_dl.h>
  59 #include <net/if_types.h>
  60 #include <net/if_media.h>
  61 #include <net/if_vlan_var.h>
  62 #include <net/bpf.h>
  63
  64 #include <netinet/in_systm.h>
  65 #include <netinet/in.h>
  66 #include <netinet/ip.h>
  67 #include <netinet/ip6.h>
  68 #include <netinet6/ip6_var.h>
  69 #include <netinet/udp.h>
  70 #include <netinet/tcp.h>
  71 #include <netinet/sctp.h>
  72
  73 #include <machine/bus.h>
  74 #include <machine/resource.h>
  75 #include <sys/bus.h>
  76 #include <sys/rman.h>
  77
  78 #include <dev/pci/pcivar.h>
  79 #include <dev/pci/pcireg.h>
  80
  81 #include "opt_inet.h"
  82 #include "opt_inet6.h"
  83
  84 #include <sys/selinfo.h>
  85 #include <net/netmap.h>
  86 #include <dev/netmap/netmap_kern.h>
  87 #include <net/netmap_virt.h>
  88 #include <dev/netmap/netmap_mem2.h>
  89 #include <dev/virtio/network/virtio_net.h>
  90
  91 #ifndef INET
  92 #error "INET not defined, cannot support offloadings"
  93 #endif
  94
  95 #if __FreeBSD_version >= 1100000
  96 static uint64_t ptnet_get_counter(if_t, ift_counter);
  97 #else
  98 typedef struct ifnet *if_t;
  99 #define if_getsoftc(_ifp)   (_ifp)->if_softc
 100 #endif
 101
 102 //#define PTNETMAP_STATS
 103 //#define DEBUG
 104 #ifdef DEBUG
 105 #define DBG(x) x
 106 #else   /* !DEBUG */
 107 #define DBG(x)
 108 #endif  /* !DEBUG */
 109
 110 extern int ptnet_vnet_hdr; /* Tunable parameter */
 111
 112 struct ptnet_softc;
 113
 114 struct ptnet_queue_stats {
 115         uint64_t        packets; /* if_[io]packets */
 116         uint64_t        bytes;   /* if_[io]bytes */
 117         uint64_t        errors;  /* if_[io]errors */
 118         uint64_t        iqdrops; /* if_iqdrops */
 119         uint64_t        mcasts;  /* if_[io]mcasts */
 120 #ifdef PTNETMAP_STATS
 121         uint64_t        intrs;
 122         uint64_t        kicks;
 123 #endif /* PTNETMAP_STATS */
 124 };
 125
 126 struct ptnet_queue {
 127         struct ptnet_softc              *sc;
 128         struct                          resource *irq;
 129         void                            *cookie;
 130         int                             kring_id;
 131         struct nm_csb_atok              *atok;
 132         struct nm_csb_ktoa              *ktoa;
 133         unsigned int                    kick;
 134         struct mtx                      lock;
 135         struct buf_ring                 *bufring; /* for TX queues */
 136         struct ptnet_queue_stats        stats;
 137 #ifdef PTNETMAP_STATS
 138         struct ptnet_queue_stats        last_stats;
 139 #endif /* PTNETMAP_STATS */
 140         struct taskqueue                *taskq;
 141         struct task                     task;
 142         char                            lock_name[16];
 143 };
 144
 145 #define PTNET_Q_LOCK(_pq)       mtx_lock(&(_pq)->lock)
 146 #define PTNET_Q_TRYLOCK(_pq)    mtx_trylock(&(_pq)->lock)
 147 #define PTNET_Q_UNLOCK(_pq)     mtx_unlock(&(_pq)->lock)
 148
 149 struct ptnet_softc {
 150         device_t                dev;
 151         if_t                    ifp;
 152         struct ifmedia          media;
 153         struct mtx              lock;
 154         char                    lock_name[16];
 155         char                    hwaddr[ETHER_ADDR_LEN];
 156
 157         /* Mirror of PTFEAT register. */
 158         uint32_t                ptfeatures;
 159         unsigned int            vnet_hdr_len;
 160
 161         /* PCI BARs support. */
 162         struct resource         *iomem;
 163         struct resource         *msix_mem;
 164
 165         unsigned int            num_rings;
 166         unsigned int            num_tx_rings;
 167         struct ptnet_queue      *queues;
 168         struct ptnet_queue      *rxqueues;
 169         struct nm_csb_atok      *csb_gh;
 170         struct nm_csb_ktoa      *csb_hg;
 171
 172         unsigned int            min_tx_space;
 173
 174         struct netmap_pt_guest_adapter *ptna;
 175
 176         struct callout          tick;
 177 #ifdef PTNETMAP_STATS
 178         struct timeval          last_ts;
 179 #endif /* PTNETMAP_STATS */
 180 };
 181
 182 #define PTNET_CORE_LOCK(_sc)    mtx_lock(&(_sc)->lock)
 183 #define PTNET_CORE_UNLOCK(_sc)  mtx_unlock(&(_sc)->lock)
 184
 185 static int      ptnet_probe(device_t);
 186 static int      ptnet_attach(device_t);
 187 static int      ptnet_detach(device_t);
 188 static int      ptnet_suspend(device_t);
 189 static int      ptnet_resume(device_t);
 190 static int      ptnet_shutdown(device_t);
 191
 192 static void     ptnet_init(void *opaque);
 193 static int      ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data);
 194 static int      ptnet_init_locked(struct ptnet_softc *sc);
 195 static int      ptnet_stop(struct ptnet_softc *sc);
 196 static int      ptnet_transmit(if_t ifp, struct mbuf *m);
 197 static int      ptnet_drain_transmit_queue(struct ptnet_queue *pq,
 198                                            unsigned int budget,
 199                                            bool may_resched);
 200 static void     ptnet_qflush(if_t ifp);
 201 static void     ptnet_tx_task(void *context, int pending);
 202
 203 static int      ptnet_media_change(if_t ifp);
 204 static void     ptnet_media_status(if_t ifp, struct ifmediareq *ifmr);
 205 #ifdef PTNETMAP_STATS
 206 static void     ptnet_tick(void *opaque);
 207 #endif
 208
 209 static int      ptnet_irqs_init(struct ptnet_softc *sc);
 210 static void     ptnet_irqs_fini(struct ptnet_softc *sc);
 211
 212 static uint32_t ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd);
 213 static int      ptnet_nm_config(struct netmap_adapter *na,
 214                                 struct nm_config_info *info);
 215 static void     ptnet_update_vnet_hdr(struct ptnet_softc *sc);
 216 static int      ptnet_nm_register(struct netmap_adapter *na, int onoff);
 217 static int      ptnet_nm_txsync(struct netmap_kring *kring, int flags);
 218 static int      ptnet_nm_rxsync(struct netmap_kring *kring, int flags);
 219 static void     ptnet_nm_intr(struct netmap_adapter *na, int onoff);
 220
 221 static void     ptnet_tx_intr(void *opaque);
 222 static void     ptnet_rx_intr(void *opaque);
 223
 224 static unsigned ptnet_rx_discard(struct netmap_kring *kring,
 225                                  unsigned int head);
 226 static int      ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget,
 227                              bool may_resched);
 228 static void     ptnet_rx_task(void *context, int pending);
 229
 230 #ifdef DEVICE_POLLING
 231 static poll_handler_t ptnet_poll;
 232 #endif
 233
 234 static device_method_t ptnet_methods[] = {
 235         DEVMETHOD(device_probe,                 ptnet_probe),
 236         DEVMETHOD(device_attach,                ptnet_attach),
 237         DEVMETHOD(device_detach,                ptnet_detach),
 238         DEVMETHOD(device_suspend,               ptnet_suspend),
 239         DEVMETHOD(device_resume,                ptnet_resume),
 240         DEVMETHOD(device_shutdown,              ptnet_shutdown),
 241         DEVMETHOD_END
 242 };
 243
 244 static driver_t ptnet_driver = {
 245         "ptnet",
 246         ptnet_methods,
 247         sizeof(struct ptnet_softc)
 248 };
 249
 250 /* We use (SI_ORDER_MIDDLE+2) here, see DEV_MODULE_ORDERED() invocation. */
 251 static devclass_t ptnet_devclass;
 252 DRIVER_MODULE_ORDERED(ptnet, pci, ptnet_driver, ptnet_devclass,
 253                       NULL, NULL, SI_ORDER_MIDDLE + 2);
 254
 255 static int
 256 ptnet_probe(device_t dev)
 257 {
 258         if (pci_get_vendor(dev) != PTNETMAP_PCI_VENDOR_ID ||
 259                 pci_get_device(dev) != PTNETMAP_PCI_NETIF_ID) {
 260                 return (ENXIO);
 261         }
 262
 263         device_set_desc(dev, "ptnet network adapter");
 264
 265         return (BUS_PROBE_DEFAULT);
 266 }
 267
 268 static inline void ptnet_kick(struct ptnet_queue *pq)
 269 {
 270 #ifdef PTNETMAP_STATS
 271         pq->stats.kicks ++;
 272 #endif /* PTNETMAP_STATS */
 273         bus_write_4(pq->sc->iomem, pq->kick, 0);
 274 }
 275
 276 #define PTNET_BUF_RING_SIZE     4096
 277 #define PTNET_RX_BUDGET         512
 278 #define PTNET_RX_BATCH          1
 279 #define PTNET_TX_BUDGET         512
 280 #define PTNET_TX_BATCH          64
 281 #define PTNET_HDR_SIZE          sizeof(struct virtio_net_hdr_mrg_rxbuf)
 282 #define PTNET_MAX_PKT_SIZE      65536
 283
 284 #define PTNET_CSUM_OFFLOAD      (CSUM_TCP | CSUM_UDP | CSUM_SCTP)
 285 #define PTNET_CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6 | CSUM_UDP_IPV6 |\
 286                                  CSUM_SCTP_IPV6)
 287 #define PTNET_ALL_OFFLOAD       (CSUM_TSO | PTNET_CSUM_OFFLOAD |\
 288                                  PTNET_CSUM_OFFLOAD_IPV6)
 289
 290 static int
 291 ptnet_attach(device_t dev)
 292 {
 293         uint32_t ptfeatures = 0;
 294         unsigned int num_rx_rings, num_tx_rings;
 295         struct netmap_adapter na_arg;
 296         unsigned int nifp_offset;
 297         struct ptnet_softc *sc;
 298         if_t ifp;
 299         uint32_t macreg;
 300         int err, rid;
 301         int i;
 302
 303         sc = device_get_softc(dev);
 304         sc->dev = dev;
 305
 306         /* Setup PCI resources. */
 307         pci_enable_busmaster(dev);
 308
 309         rid = PCIR_BAR(PTNETMAP_IO_PCI_BAR);
 310         sc->iomem = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
 311                                            RF_ACTIVE);
 312         if (sc->iomem == NULL) {
 313                 device_printf(dev, "Failed to map I/O BAR\n");
 314                 return (ENXIO);
 315         }
 316
 317         /* Negotiate features with the hypervisor. */
 318         if (ptnet_vnet_hdr) {
 319                 ptfeatures |= PTNETMAP_F_VNET_HDR;
 320         }
 321         bus_write_4(sc->iomem, PTNET_IO_PTFEAT, ptfeatures); /* wanted */
 322         ptfeatures = bus_read_4(sc->iomem, PTNET_IO_PTFEAT); /* acked */
 323         sc->ptfeatures = ptfeatures;
 324
 325         num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
 326         num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
 327         sc->num_rings = num_tx_rings + num_rx_rings;
 328         sc->num_tx_rings = num_tx_rings;
 329
 330         if (sc->num_rings * sizeof(struct nm_csb_atok) > PAGE_SIZE) {
 331                 device_printf(dev, "CSB cannot handle that many rings (%u)\n",
 332                                 sc->num_rings);
 333                 err = ENOMEM;
 334                 goto err_path;
 335         }
 336
 337         /* Allocate CSB and carry out CSB allocation protocol. */
 338         sc->csb_gh = contigmalloc(2*PAGE_SIZE, M_DEVBUF, M_NOWAIT | M_ZERO,
 339                                   (size_t)0, -1UL, PAGE_SIZE, 0);
 340         if (sc->csb_gh == NULL) {
 341                 device_printf(dev, "Failed to allocate CSB\n");
 342                 err = ENOMEM;
 343                 goto err_path;
 344         }
 345         sc->csb_hg = (struct nm_csb_ktoa *)(((char *)sc->csb_gh) + PAGE_SIZE);
 346
 347         {
 348                 /*
 349                  * We use uint64_t rather than vm_paddr_t since we
 350                  * need 64 bit addresses even on 32 bit platforms.
 351                  */
 352                 uint64_t paddr = vtophys(sc->csb_gh);
 353
 354                 /* CSB allocation protocol: write to BAH first, then
 355                  * to BAL (for both GH and HG sections). */
 356                 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH,
 357                                 (paddr >> 32) & 0xffffffff);
 358                 bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL,
 359                                 paddr & 0xffffffff);
 360                 paddr = vtophys(sc->csb_hg);
 361                 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH,
 362                                 (paddr >> 32) & 0xffffffff);
 363                 bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL,
 364                                 paddr & 0xffffffff);
 365         }
 366
 367         /* Allocate and initialize per-queue data structures. */
 368         sc->queues = malloc(sizeof(struct ptnet_queue) * sc->num_rings,
 369                             M_DEVBUF, M_NOWAIT | M_ZERO);
 370         if (sc->queues == NULL) {
 371                 err = ENOMEM;
 372                 goto err_path;
 373         }
 374         sc->rxqueues = sc->queues + num_tx_rings;
 375
 376         for (i = 0; i < sc->num_rings; i++) {
 377                 struct ptnet_queue *pq = sc->queues + i;
 378
 379                 pq->sc = sc;
 380                 pq->kring_id = i;
 381                 pq->kick = PTNET_IO_KICK_BASE + 4 * i;
 382                 pq->atok = sc->csb_gh + i;
 383                 pq->ktoa = sc->csb_hg + i;
 384                 snprintf(pq->lock_name, sizeof(pq->lock_name), "%s-%d",
 385                          device_get_nameunit(dev), i);
 386                 mtx_init(&pq->lock, pq->lock_name, NULL, MTX_DEF);
 387                 if (i >= num_tx_rings) {
 388                         /* RX queue: fix kring_id. */
 389                         pq->kring_id -= num_tx_rings;
 390                 } else {
 391                         /* TX queue: allocate buf_ring. */
 392                         pq->bufring = buf_ring_alloc(PTNET_BUF_RING_SIZE,
 393                                                 M_DEVBUF, M_NOWAIT, &pq->lock);
 394                         if (pq->bufring == NULL) {
 395                                 err = ENOMEM;
 396                                 goto err_path;
 397                         }
 398                 }
 399         }
 400
 401         sc->min_tx_space = 64; /* Safe initial value. */
 402
 403         err = ptnet_irqs_init(sc);
 404         if (err) {
 405                 goto err_path;
 406         }
 407
 408         /* Setup Ethernet interface. */
 409         sc->ifp = ifp = if_alloc(IFT_ETHER);
 410         if (ifp == NULL) {
 411                 device_printf(dev, "Failed to allocate ifnet\n");
 412                 err = ENOMEM;
 413                 goto err_path;
 414         }
 415
 416         if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 417         ifp->if_baudrate = IF_Gbps(10);
 418         ifp->if_softc = sc;
 419         ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX;
 420         ifp->if_init = ptnet_init;
 421         ifp->if_ioctl = ptnet_ioctl;
 422 #if __FreeBSD_version >= 1100000
 423         ifp->if_get_counter = ptnet_get_counter;
 424 #endif
 425         ifp->if_transmit = ptnet_transmit;
 426         ifp->if_qflush = ptnet_qflush;
 427
 428         ifmedia_init(&sc->media, IFM_IMASK, ptnet_media_change,
 429                      ptnet_media_status);
 430         ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX, 0, NULL);
 431         ifmedia_set(&sc->media, IFM_ETHER | IFM_10G_T | IFM_FDX);
 432
 433         macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_HI);
 434         sc->hwaddr[0] = (macreg >> 8) & 0xff;
 435         sc->hwaddr[1] = macreg & 0xff;
 436         macreg = bus_read_4(sc->iomem, PTNET_IO_MAC_LO);
 437         sc->hwaddr[2] = (macreg >> 24) & 0xff;
 438         sc->hwaddr[3] = (macreg >> 16) & 0xff;
 439         sc->hwaddr[4] = (macreg >> 8) & 0xff;
 440         sc->hwaddr[5] = macreg & 0xff;
 441
 442         ether_ifattach(ifp, sc->hwaddr);
 443
 444         ifp->if_hdrlen = sizeof(struct ether_vlan_header);
 445         ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;
 446
 447         if (sc->ptfeatures & PTNETMAP_F_VNET_HDR) {
 448                 /* Similarly to what the vtnet driver does, we can emulate
 449                  * VLAN offloadings by inserting and removing the 802.1Q
 450                  * header during transmit and receive. We are then able
 451                  * to do checksum offloading of VLAN frames. */
 452                 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6
 453                                         | IFCAP_VLAN_HWCSUM
 454                                         | IFCAP_TSO | IFCAP_LRO
 455                                         | IFCAP_VLAN_HWTSO
 456                                         | IFCAP_VLAN_HWTAGGING;
 457         }
 458
 459         ifp->if_capenable = ifp->if_capabilities;
 460 #ifdef DEVICE_POLLING
 461         /* Don't enable polling by default. */
 462         ifp->if_capabilities |= IFCAP_POLLING;
 463 #endif
 464         snprintf(sc->lock_name, sizeof(sc->lock_name),
 465                  "%s", device_get_nameunit(dev));
 466         mtx_init(&sc->lock, sc->lock_name, "ptnet core lock", MTX_DEF);
 467         callout_init_mtx(&sc->tick, &sc->lock, 0);
 468
 469         /* Prepare a netmap_adapter struct instance to do netmap_attach(). */
 470         nifp_offset = bus_read_4(sc->iomem, PTNET_IO_NIFP_OFS);
 471         memset(&na_arg, 0, sizeof(na_arg));
 472         na_arg.ifp = ifp;
 473         na_arg.num_tx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
 474         na_arg.num_rx_desc = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
 475         na_arg.num_tx_rings = num_tx_rings;
 476         na_arg.num_rx_rings = num_rx_rings;
 477         na_arg.nm_config = ptnet_nm_config;
 478         na_arg.nm_krings_create = ptnet_nm_krings_create;
 479         na_arg.nm_krings_delete = ptnet_nm_krings_delete;
 480         na_arg.nm_dtor = ptnet_nm_dtor;
 481         na_arg.nm_intr = ptnet_nm_intr;
 482         na_arg.nm_register = ptnet_nm_register;
 483         na_arg.nm_txsync = ptnet_nm_txsync;
 484         na_arg.nm_rxsync = ptnet_nm_rxsync;
 485
 486         netmap_pt_guest_attach(&na_arg, nifp_offset,
 487                                 bus_read_4(sc->iomem, PTNET_IO_HOSTMEMID));
 488
 489         /* Now a netmap adapter for this ifp has been allocated, and it
 490          * can be accessed through NA(ifp). We also have to initialize the CSB
 491          * pointer. */
 492         sc->ptna = (struct netmap_pt_guest_adapter *)NA(ifp);
 493
 494         /* If virtio-net header was negotiated, set the virt_hdr_len field in
 495          * the netmap adapter, to inform users that this netmap adapter requires
 496          * the application to deal with the headers. */
 497         ptnet_update_vnet_hdr(sc);
 498
 499         device_printf(dev, "%s() completed\n", __func__);
 500
 501         return (0);
 502
 503 err_path:
 504         ptnet_detach(dev);
 505         return err;
 506 }
 507
 508 /* Stop host sync-kloop if it was running. */
 509 static void
 510 ptnet_device_shutdown(struct ptnet_softc *sc)
 511 {
 512         ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
 513         bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAH, 0);
 514         bus_write_4(sc->iomem, PTNET_IO_CSB_GH_BAL, 0);
 515         bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAH, 0);
 516         bus_write_4(sc->iomem, PTNET_IO_CSB_HG_BAL, 0);
 517 }
 518
 519 static int
 520 ptnet_detach(device_t dev)
 521 {
 522         struct ptnet_softc *sc = device_get_softc(dev);
 523         int i;
 524
 525         ptnet_device_shutdown(sc);
 526
 527 #ifdef DEVICE_POLLING
 528         if (sc->ifp->if_capenable & IFCAP_POLLING) {
 529                 ether_poll_deregister(sc->ifp);
 530         }
 531 #endif
 532         callout_drain(&sc->tick);
 533
 534         if (sc->queues) {
 535                 /* Drain taskqueues before calling if_detach. */
 536                 for (i = 0; i < sc->num_rings; i++) {
 537                         struct ptnet_queue *pq = sc->queues + i;
 538
 539                         if (pq->taskq) {
 540                                 taskqueue_drain(pq->taskq, &pq->task);
 541                         }
 542                 }
 543         }
 544
 545         if (sc->ifp) {
 546                 ether_ifdetach(sc->ifp);
 547
 548                 /* Uninitialize netmap adapters for this device. */
 549                 netmap_detach(sc->ifp);
 550
 551                 ifmedia_removeall(&sc->media);
 552                 if_free(sc->ifp);
 553                 sc->ifp = NULL;
 554         }
 555
 556         ptnet_irqs_fini(sc);
 557
 558         if (sc->csb_gh) {
 559                 contigfree(sc->csb_gh, 2*PAGE_SIZE, M_DEVBUF);
 560                 sc->csb_gh = NULL;
 561                 sc->csb_hg = NULL;
 562         }
 563
 564         if (sc->queues) {
 565                 for (i = 0; i < sc->num_rings; i++) {
 566                         struct ptnet_queue *pq = sc->queues + i;
 567
 568                         if (mtx_initialized(&pq->lock)) {
 569                                 mtx_destroy(&pq->lock);
 570                         }
 571                         if (pq->bufring != NULL) {
 572                                 buf_ring_free(pq->bufring, M_DEVBUF);
 573                         }
 574                 }
 575                 free(sc->queues, M_DEVBUF);
 576                 sc->queues = NULL;
 577         }
 578
 579         if (sc->iomem) {
 580                 bus_release_resource(dev, SYS_RES_IOPORT,
 581                                      PCIR_BAR(PTNETMAP_IO_PCI_BAR), sc->iomem);
 582                 sc->iomem = NULL;
 583         }
 584
 585         mtx_destroy(&sc->lock);
 586
 587         device_printf(dev, "%s() completed\n", __func__);
 588
 589         return (0);
 590 }
 591
 592 static int
 593 ptnet_suspend(device_t dev)
 594 {
 595         struct ptnet_softc *sc = device_get_softc(dev);
 596
 597         (void)sc;
 598
 599         return (0);
 600 }
 601
 602 static int
 603 ptnet_resume(device_t dev)
 604 {
 605         struct ptnet_softc *sc = device_get_softc(dev);
 606
 607         (void)sc;
 608
 609         return (0);
 610 }
 611
 612 static int
 613 ptnet_shutdown(device_t dev)
 614 {
 615         struct ptnet_softc *sc = device_get_softc(dev);
 616
 617         ptnet_device_shutdown(sc);
 618
 619         return (0);
 620 }
 621
 622 static int
 623 ptnet_irqs_init(struct ptnet_softc *sc)
 624 {
 625         int rid = PCIR_BAR(PTNETMAP_MSIX_PCI_BAR);
 626         int nvecs = sc->num_rings;
 627         device_t dev = sc->dev;
 628         int err = ENOSPC;
 629         int cpu_cur;
 630         int i;
 631
 632         if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0)  {
 633                 device_printf(dev, "Could not find MSI-X capability\n");
 634                 return (ENXIO);
 635         }
 636
 637         sc->msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 638                                               &rid, RF_ACTIVE);
 639         if (sc->msix_mem == NULL) {
 640                 device_printf(dev, "Failed to allocate MSIX PCI BAR\n");
 641                 return (ENXIO);
 642         }
 643
 644         if (pci_msix_count(dev) < nvecs) {
 645                 device_printf(dev, "Not enough MSI-X vectors\n");
 646                 goto err_path;
 647         }
 648
 649         err = pci_alloc_msix(dev, &nvecs);
 650         if (err) {
 651                 device_printf(dev, "Failed to allocate MSI-X vectors\n");
 652                 goto err_path;
 653         }
 654
 655         for (i = 0; i < nvecs; i++) {
 656                 struct ptnet_queue *pq = sc->queues + i;
 657
 658                 rid = i + 1;
 659                 pq->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 660                                                  RF_ACTIVE);
 661                 if (pq->irq == NULL) {
 662                         device_printf(dev, "Failed to allocate interrupt "
 663                                            "for queue #%d\n", i);
 664                         err = ENOSPC;
 665                         goto err_path;
 666                 }
 667         }
 668
 669         cpu_cur = CPU_FIRST();
 670         for (i = 0; i < nvecs; i++) {
 671                 struct ptnet_queue *pq = sc->queues + i;
 672                 void (*handler)(void *) = ptnet_tx_intr;
 673
 674                 if (i >= sc->num_tx_rings) {
 675                         handler = ptnet_rx_intr;
 676                 }
 677                 err = bus_setup_intr(dev, pq->irq, INTR_TYPE_NET | INTR_MPSAFE,
 678                                      NULL /* intr_filter */, handler,
 679                                      pq, &pq->cookie);
 680                 if (err) {
 681                         device_printf(dev, "Failed to register intr handler "
 682                                            "for queue #%d\n", i);
 683                         goto err_path;
 684                 }
 685
 686                 bus_describe_intr(dev, pq->irq, pq->cookie, "q%d", i);
 687 #if 0
 688                 bus_bind_intr(sc->dev, pq->irq, cpu_cur);
 689 #endif
 690                 cpu_cur = CPU_NEXT(cpu_cur);
 691         }
 692
 693         device_printf(dev, "Allocated %d MSI-X vectors\n", nvecs);
 694
 695         cpu_cur = CPU_FIRST();
 696         for (i = 0; i < nvecs; i++) {
 697                 struct ptnet_queue *pq = sc->queues + i;
 698                 static void (*handler)(void *context, int pending);
 699
 700                 handler = (i < sc->num_tx_rings) ? ptnet_tx_task : ptnet_rx_task;
 701
 702                 TASK_INIT(&pq->task, 0, handler, pq);
 703                 pq->taskq = taskqueue_create_fast("ptnet_queue", M_NOWAIT,
 704                                         taskqueue_thread_enqueue, &pq->taskq);
 705                 taskqueue_start_threads(&pq->taskq, 1, PI_NET, "%s-pq-%d",
 706                                         device_get_nameunit(sc->dev), cpu_cur);
 707                 cpu_cur = CPU_NEXT(cpu_cur);
 708         }
 709
 710         return 0;
 711 err_path:
 712         ptnet_irqs_fini(sc);
 713         return err;
 714 }
 715
 716 static void
 717 ptnet_irqs_fini(struct ptnet_softc *sc)
 718 {
 719         device_t dev = sc->dev;
 720         int i;
 721
 722         for (i = 0; i < sc->num_rings; i++) {
 723                 struct ptnet_queue *pq = sc->queues + i;
 724
 725                 if (pq->taskq) {
 726                         taskqueue_free(pq->taskq);
 727                         pq->taskq = NULL;
 728                 }
 729
 730                 if (pq->cookie) {
 731                         bus_teardown_intr(dev, pq->irq, pq->cookie);
 732                         pq->cookie = NULL;
 733                 }
 734
 735                 if (pq->irq) {
 736                         bus_release_resource(dev, SYS_RES_IRQ, i + 1, pq->irq);
 737                         pq->irq = NULL;
 738                 }
 739         }
 740
 741         if (sc->msix_mem) {
 742                 pci_release_msi(dev);
 743
 744                 bus_release_resource(dev, SYS_RES_MEMORY,
 745                                      PCIR_BAR(PTNETMAP_MSIX_PCI_BAR),
 746                                      sc->msix_mem);
 747                 sc->msix_mem = NULL;
 748         }
 749 }
 750
 751 static void
 752 ptnet_init(void *opaque)
 753 {
 754         struct ptnet_softc *sc = opaque;
 755
 756         PTNET_CORE_LOCK(sc);
 757         ptnet_init_locked(sc);
 758         PTNET_CORE_UNLOCK(sc);
 759 }
 760
 761 static int
 762 ptnet_ioctl(if_t ifp, u_long cmd, caddr_t data)
 763 {
 764         struct ptnet_softc *sc = if_getsoftc(ifp);
 765         device_t dev = sc->dev;
 766         struct ifreq *ifr = (struct ifreq *)data;
 767         int mask __unused, err = 0;
 768
 769         switch (cmd) {
 770         case SIOCSIFFLAGS:
 771                 device_printf(dev, "SIOCSIFFLAGS %x\n", ifp->if_flags);
 772                 PTNET_CORE_LOCK(sc);
 773                 if (ifp->if_flags & IFF_UP) {
 774                         /* Network stack wants the iff to be up. */
 775                         err = ptnet_init_locked(sc);
 776                 } else {
 777                         /* Network stack wants the iff to be down. */
 778                         err = ptnet_stop(sc);
 779                 }
 780                 /* We don't need to do nothing to support IFF_PROMISC,
 781                  * since that is managed by the backend port. */
 782                 PTNET_CORE_UNLOCK(sc);
 783                 break;
 784
 785         case SIOCSIFCAP:
 786                 device_printf(dev, "SIOCSIFCAP %x %x\n",
 787                               ifr->ifr_reqcap, ifp->if_capenable);
 788                 mask = ifr->ifr_reqcap ^ ifp->if_capenable;
 789 #ifdef DEVICE_POLLING
 790                 if (mask & IFCAP_POLLING) {
 791                         struct ptnet_queue *pq;
 792                         int i;
 793
 794                         if (ifr->ifr_reqcap & IFCAP_POLLING) {
 795                                 err = ether_poll_register(ptnet_poll, ifp);
 796                                 if (err) {
 797                                         break;
 798                                 }
 799                                 /* Stop queues and sync with taskqueues. */
 800                                 ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 801                                 for (i = 0; i < sc->num_rings; i++) {
 802                                         pq = sc-> queues + i;
 803                                         /* Make sure the worker sees the
 804                                          * IFF_DRV_RUNNING down. */
 805                                         PTNET_Q_LOCK(pq);
 806                                         pq->atok->appl_need_kick = 0;
 807                                         PTNET_Q_UNLOCK(pq);
 808                                         /* Wait for rescheduling to finish. */
 809                                         if (pq->taskq) {
 810                                                 taskqueue_drain(pq->taskq,
 811                                                                 &pq->task);
 812                                         }
 813                                 }
 814                                 ifp->if_drv_flags |= IFF_DRV_RUNNING;
 815                         } else {
 816                                 err = ether_poll_deregister(ifp);
 817                                 for (i = 0; i < sc->num_rings; i++) {
 818                                         pq = sc-> queues + i;
 819                                         PTNET_Q_LOCK(pq);
 820                                         pq->atok->appl_need_kick = 1;
 821                                         PTNET_Q_UNLOCK(pq);
 822                                 }
 823                         }
 824                 }
 825 #endif  /* DEVICE_POLLING */
 826                 ifp->if_capenable = ifr->ifr_reqcap;
 827                 break;
 828
 829         case SIOCSIFMTU:
 830                 /* We support any reasonable MTU. */
 831                 if (ifr->ifr_mtu < ETHERMIN ||
 832                                 ifr->ifr_mtu > PTNET_MAX_PKT_SIZE) {
 833                         err = EINVAL;
 834                 } else {
 835                         PTNET_CORE_LOCK(sc);
 836                         ifp->if_mtu = ifr->ifr_mtu;
 837                         PTNET_CORE_UNLOCK(sc);
 838                 }
 839                 break;
 840
 841         case SIOCSIFMEDIA:
 842         case SIOCGIFMEDIA:
 843                 err = ifmedia_ioctl(ifp, ifr, &sc->media, cmd);
 844                 break;
 845
 846         default:
 847                 err = ether_ioctl(ifp, cmd, data);
 848                 break;
 849         }
 850
 851         return err;
 852 }
 853
 854 static int
 855 ptnet_init_locked(struct ptnet_softc *sc)
 856 {
 857         if_t ifp = sc->ifp;
 858         struct netmap_adapter *na_dr = &sc->ptna->dr.up;
 859         struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
 860         unsigned int nm_buf_size;
 861         int ret;
 862
 863         if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 864                 return 0; /* nothing to do */
 865         }
 866
 867         device_printf(sc->dev, "%s\n", __func__);
 868
 869         /* Translate offload capabilities according to if_capenable. */
 870         ifp->if_hwassist = 0;
 871         if (ifp->if_capenable & IFCAP_TXCSUM)
 872                 ifp->if_hwassist |= PTNET_CSUM_OFFLOAD;
 873         if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
 874                 ifp->if_hwassist |= PTNET_CSUM_OFFLOAD_IPV6;
 875         if (ifp->if_capenable & IFCAP_TSO4)
 876                 ifp->if_hwassist |= CSUM_IP_TSO;
 877         if (ifp->if_capenable & IFCAP_TSO6)
 878                 ifp->if_hwassist |= CSUM_IP6_TSO;
 879
 880         /*
 881          * Prepare the interface for netmap mode access.
 882          */
 883         netmap_update_config(na_dr);
 884
 885         ret = netmap_mem_finalize(na_dr->nm_mem, na_dr);
 886         if (ret) {
 887                 device_printf(sc->dev, "netmap_mem_finalize() failed\n");
 888                 return ret;
 889         }
 890
 891         if (sc->ptna->backend_users == 0) {
 892                 ret = ptnet_nm_krings_create(na_nm);
 893                 if (ret) {
 894                         device_printf(sc->dev, "ptnet_nm_krings_create() "
 895                                                "failed\n");
 896                         goto err_mem_finalize;
 897                 }
 898
 899                 ret = netmap_mem_rings_create(na_dr);
 900                 if (ret) {
 901                         device_printf(sc->dev, "netmap_mem_rings_create() "
 902                                                "failed\n");
 903                         goto err_rings_create;
 904                 }
 905
 906                 ret = netmap_mem_get_lut(na_dr->nm_mem, &na_dr->na_lut);
 907                 if (ret) {
 908                         device_printf(sc->dev, "netmap_mem_get_lut() "
 909                                                "failed\n");
 910                         goto err_get_lut;
 911                 }
 912         }
 913
 914         ret = ptnet_nm_register(na_dr, 1 /* on */);
 915         if (ret) {
 916                 goto err_register;
 917         }
 918
 919         nm_buf_size = NETMAP_BUF_SIZE(na_dr);
 920
 921         KASSERT(nm_buf_size > 0, ("Invalid netmap buffer size"));
 922         sc->min_tx_space = PTNET_MAX_PKT_SIZE / nm_buf_size + 2;
 923         device_printf(sc->dev, "%s: min_tx_space = %u\n", __func__,
 924                       sc->min_tx_space);
 925 #ifdef PTNETMAP_STATS
 926         callout_reset(&sc->tick, hz, ptnet_tick, sc);
 927 #endif
 928
 929         ifp->if_drv_flags |= IFF_DRV_RUNNING;
 930
 931         return 0;
 932
 933 err_register:
 934         memset(&na_dr->na_lut, 0, sizeof(na_dr->na_lut));
 935 err_get_lut:
 936         netmap_mem_rings_delete(na_dr);
 937 err_rings_create:
 938         ptnet_nm_krings_delete(na_nm);
 939 err_mem_finalize:
 940         netmap_mem_deref(na_dr->nm_mem, na_dr);
 941
 942         return ret;
 943 }
 944
 945 /* To be called under core lock. */
 946 static int
 947 ptnet_stop(struct ptnet_softc *sc)
 948 {
 949         if_t ifp = sc->ifp;
 950         struct netmap_adapter *na_dr = &sc->ptna->dr.up;
 951         struct netmap_adapter *na_nm = &sc->ptna->hwup.up;
 952         int i;
 953
 954         device_printf(sc->dev, "%s\n", __func__);
 955
 956         if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
 957                 return 0; /* nothing to do */
 958         }
 959
 960         /* Clear the driver-ready flag, and synchronize with all the queues,
 961          * so that after this loop we are sure nobody is working anymore with
 962          * the device. This scheme is taken from the vtnet driver. */
 963         ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 964         callout_stop(&sc->tick);
 965         for (i = 0; i < sc->num_rings; i++) {
 966                 PTNET_Q_LOCK(sc->queues + i);
 967                 PTNET_Q_UNLOCK(sc->queues + i);
 968         }
 969
 970         ptnet_nm_register(na_dr, 0 /* off */);
 971
 972         if (sc->ptna->backend_users == 0) {
 973                 netmap_mem_rings_delete(na_dr);
 974                 ptnet_nm_krings_delete(na_nm);
 975         }
 976         netmap_mem_deref(na_dr->nm_mem, na_dr);
 977
 978         return 0;
 979 }
 980
 981 static void
 982 ptnet_qflush(if_t ifp)
 983 {
 984         struct ptnet_softc *sc = if_getsoftc(ifp);
 985         int i;
 986
 987         /* Flush all the bufrings and do the interface flush. */
 988         for (i = 0; i < sc->num_rings; i++) {
 989                 struct ptnet_queue *pq = sc->queues + i;
 990                 struct mbuf *m;
 991
 992                 PTNET_Q_LOCK(pq);
 993                 if (pq->bufring) {
 994                         while ((m = buf_ring_dequeue_sc(pq->bufring))) {
 995                                 m_freem(m);
 996                         }
 997                 }
 998                 PTNET_Q_UNLOCK(pq);
 999         }
1000
1001         if_qflush(ifp);
1002 }
1003
1004 static int
1005 ptnet_media_change(if_t ifp)
1006 {
1007         struct ptnet_softc *sc = if_getsoftc(ifp);
1008         struct ifmedia *ifm = &sc->media;
1009
1010         if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) {
1011                 return EINVAL;
1012         }
1013
1014         return 0;
1015 }
1016
1017 #if __FreeBSD_version >= 1100000
1018 static uint64_t
1019 ptnet_get_counter(if_t ifp, ift_counter cnt)
1020 {
1021         struct ptnet_softc *sc = if_getsoftc(ifp);
1022         struct ptnet_queue_stats stats[2];
1023         int i;
1024
1025         /* Accumulate statistics over the queues. */
1026         memset(stats, 0, sizeof(stats));
1027         for (i = 0; i < sc->num_rings; i++) {
1028                 struct ptnet_queue *pq = sc->queues + i;
1029                 int idx = (i < sc->num_tx_rings) ? 0 : 1;
1030
1031                 stats[idx].packets      += pq->stats.packets;
1032                 stats[idx].bytes        += pq->stats.bytes;
1033                 stats[idx].errors       += pq->stats.errors;
1034                 stats[idx].iqdrops      += pq->stats.iqdrops;
1035                 stats[idx].mcasts       += pq->stats.mcasts;
1036         }
1037
1038         switch (cnt) {
1039         case IFCOUNTER_IPACKETS:
1040                 return (stats[1].packets);
1041         case IFCOUNTER_IQDROPS:
1042                 return (stats[1].iqdrops);
1043         case IFCOUNTER_IERRORS:
1044                 return (stats[1].errors);
1045         case IFCOUNTER_OPACKETS:
1046                 return (stats[0].packets);
1047         case IFCOUNTER_OBYTES:
1048                 return (stats[0].bytes);
1049         case IFCOUNTER_OMCASTS:
1050                 return (stats[0].mcasts);
1051         default:
1052                 return (if_get_counter_default(ifp, cnt));
1053         }
1054 }
1055 #endif
1056
1057
1058 #ifdef PTNETMAP_STATS
1059 /* Called under core lock. */
1060 static void
1061 ptnet_tick(void *opaque)
1062 {
1063         struct ptnet_softc *sc = opaque;
1064         int i;
1065
1066         for (i = 0; i < sc->num_rings; i++) {
1067                 struct ptnet_queue *pq = sc->queues + i;
1068                 struct ptnet_queue_stats cur = pq->stats;
1069                 struct timeval now;
1070                 unsigned int delta;
1071
1072                 microtime(&now);
1073                 delta = now.tv_usec - sc->last_ts.tv_usec +
1074                         (now.tv_sec - sc->last_ts.tv_sec) * 1000000;
1075                 delta /= 1000; /* in milliseconds */
1076
1077                 if (delta == 0)
1078                         continue;
1079
1080                 device_printf(sc->dev, "#%d[%u ms]:pkts %lu, kicks %lu, "
1081                               "intr %lu\n", i, delta,
1082                               (cur.packets - pq->last_stats.packets),
1083                               (cur.kicks - pq->last_stats.kicks),
1084                               (cur.intrs - pq->last_stats.intrs));
1085                 pq->last_stats = cur;
1086         }
1087         microtime(&sc->last_ts);
1088         callout_schedule(&sc->tick, hz);
1089 }
1090 #endif /* PTNETMAP_STATS */
1091
1092 static void
1093 ptnet_media_status(if_t ifp, struct ifmediareq *ifmr)
1094 {
1095         /* We are always active, as the backend netmap port is
1096          * always open in netmap mode. */
1097         ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE;
1098         ifmr->ifm_active = IFM_ETHER | IFM_10G_T | IFM_FDX;
1099 }
1100
1101 static uint32_t
1102 ptnet_nm_ptctl(struct ptnet_softc *sc, uint32_t cmd)
1103 {
1104         /*
1105          * Write a command and read back error status,
1106          * with zero meaning success.
1107          */
1108         bus_write_4(sc->iomem, PTNET_IO_PTCTL, cmd);
1109         return bus_read_4(sc->iomem, PTNET_IO_PTCTL);
1110 }
1111
1112 static int
1113 ptnet_nm_config(struct netmap_adapter *na, struct nm_config_info *info)
1114 {
1115         struct ptnet_softc *sc = if_getsoftc(na->ifp);
1116
1117         info->num_tx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_RINGS);
1118         info->num_rx_rings = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_RINGS);
1119         info->num_tx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_TX_SLOTS);
1120         info->num_rx_descs = bus_read_4(sc->iomem, PTNET_IO_NUM_RX_SLOTS);
1121         info->rx_buf_maxsize = NETMAP_BUF_SIZE(na);
1122
1123         device_printf(sc->dev, "txr %u, rxr %u, txd %u, rxd %u, rxbufsz %u\n",
1124                         info->num_tx_rings, info->num_rx_rings,
1125                         info->num_tx_descs, info->num_rx_descs,
1126                         info->rx_buf_maxsize);
1127
1128         return 0;
1129 }
1130
1131 static void
1132 ptnet_sync_from_csb(struct ptnet_softc *sc, struct netmap_adapter *na)
1133 {
1134         int i;
1135
1136         /* Sync krings from the host, reading from
1137          * CSB. */
1138         for (i = 0; i < sc->num_rings; i++) {
1139                 struct nm_csb_atok *atok = sc->queues[i].atok;
1140                 struct nm_csb_ktoa *ktoa = sc->queues[i].ktoa;
1141                 struct netmap_kring *kring;
1142
1143                 if (i < na->num_tx_rings) {
1144                         kring = na->tx_rings[i];
1145                 } else {
1146                         kring = na->rx_rings[i - na->num_tx_rings];
1147                 }
1148                 kring->rhead = kring->ring->head = atok->head;
1149                 kring->rcur = kring->ring->cur = atok->cur;
1150                 kring->nr_hwcur = ktoa->hwcur;
1151                 kring->nr_hwtail = kring->rtail =
1152                         kring->ring->tail = ktoa->hwtail;
1153
1154                 ND("%d,%d: csb {hc %u h %u c %u ht %u}", t, i,
1155                    ktoa->hwcur, atok->head, atok->cur,
1156                    ktoa->hwtail);
1157                 ND("%d,%d: kring {hc %u rh %u rc %u h %u c %u ht %u rt %u t %u}",
1158                    t, i, kring->nr_hwcur, kring->rhead, kring->rcur,
1159                    kring->ring->head, kring->ring->cur, kring->nr_hwtail,
1160                    kring->rtail, kring->ring->tail);
1161         }
1162 }
1163
1164 static void
1165 ptnet_update_vnet_hdr(struct ptnet_softc *sc)
1166 {
1167         unsigned int wanted_hdr_len = ptnet_vnet_hdr ? PTNET_HDR_SIZE : 0;
1168
1169         bus_write_4(sc->iomem, PTNET_IO_VNET_HDR_LEN, wanted_hdr_len);
1170         sc->vnet_hdr_len = bus_read_4(sc->iomem, PTNET_IO_VNET_HDR_LEN);
1171         sc->ptna->hwup.up.virt_hdr_len = sc->vnet_hdr_len;
1172 }
1173
1174 static int
1175 ptnet_nm_register(struct netmap_adapter *na, int onoff)
1176 {
1177         /* device-specific */
1178         if_t ifp = na->ifp;
1179         struct ptnet_softc *sc = if_getsoftc(ifp);
1180         int native = (na == &sc->ptna->hwup.up);
1181         struct ptnet_queue *pq;
1182         enum txrx t;
1183         int ret = 0;
1184         int i;
1185
1186         if (!onoff) {
1187                 sc->ptna->backend_users--;
1188         }
1189
1190         /* If this is the last netmap client, guest interrupt enable flags may
1191          * be in arbitrary state. Since these flags are going to be used also
1192          * by the netdevice driver, we have to make sure to start with
1193          * notifications enabled. Also, schedule NAPI to flush pending packets
1194          * in the RX rings, since we will not receive further interrupts
1195          * until these will be processed. */
1196         if (native && !onoff && na->active_fds == 0) {
1197                 D("Exit netmap mode, re-enable interrupts");
1198                 for (i = 0; i < sc->num_rings; i++) {
1199                         pq = sc->queues + i;
1200                         pq->atok->appl_need_kick = 1;
1201                 }
1202         }
1203
1204         if (onoff) {
1205                 if (sc->ptna->backend_users == 0) {
1206                         /* Initialize notification enable fields in the CSB. */
1207                         for (i = 0; i < sc->num_rings; i++) {
1208                                 pq = sc->queues + i;
1209                                 pq->ktoa->kern_need_kick = 1;
1210                                 pq->atok->appl_need_kick =
1211                                         (!(ifp->if_capenable & IFCAP_POLLING)
1212                                                 && i >= sc->num_tx_rings);
1213                         }
1214
1215                         /* Set the virtio-net header length. */
1216                         ptnet_update_vnet_hdr(sc);
1217
1218                         /* Make sure the host adapter passed through is ready
1219                          * for txsync/rxsync. */
1220                         ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_CREATE);
1221                         if (ret) {
1222                                 return ret;
1223                         }
1224
1225                         /* Align the guest krings and rings to the state stored
1226                          * in the CSB. */
1227                         ptnet_sync_from_csb(sc, na);
1228                 }
1229
1230                 /* If not native, don't call nm_set_native_flags, since we don't want
1231                  * to replace if_transmit method, nor set NAF_NETMAP_ON */
1232                 if (native) {
1233                         for_rx_tx(t) {
1234                                 for (i = 0; i <= nma_get_nrings(na, t); i++) {
1235                                         struct netmap_kring *kring = NMR(na, t)[i];
1236
1237                                         if (nm_kring_pending_on(kring)) {
1238                                                 kring->nr_mode = NKR_NETMAP_ON;
1239                                         }
1240                                 }
1241                         }
1242                         nm_set_native_flags(na);
1243                 }
1244
1245         } else {
1246                 if (native) {
1247                         nm_clear_native_flags(na);
1248                         for_rx_tx(t) {
1249                                 for (i = 0; i <= nma_get_nrings(na, t); i++) {
1250                                         struct netmap_kring *kring = NMR(na, t)[i];
1251
1252                                         if (nm_kring_pending_off(kring)) {
1253                                                 kring->nr_mode = NKR_NETMAP_OFF;
1254                                         }
1255                                 }
1256                         }
1257                 }
1258
1259                 if (sc->ptna->backend_users == 0) {
1260                         ret = ptnet_nm_ptctl(sc, PTNETMAP_PTCTL_DELETE);
1261                 }
1262         }
1263
1264         if (onoff) {
1265                 sc->ptna->backend_users++;
1266         }
1267
1268         return ret;
1269 }
1270
1271 static int
1272 ptnet_nm_txsync(struct netmap_kring *kring, int flags)
1273 {
1274         struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1275         struct ptnet_queue *pq = sc->queues + kring->ring_id;
1276         bool notify;
1277
1278         notify = netmap_pt_guest_txsync(pq->atok, pq->ktoa, kring, flags);
1279         if (notify) {
1280                 ptnet_kick(pq);
1281         }
1282
1283         return 0;
1284 }
1285
1286 static int
1287 ptnet_nm_rxsync(struct netmap_kring *kring, int flags)
1288 {
1289         struct ptnet_softc *sc = if_getsoftc(kring->na->ifp);
1290         struct ptnet_queue *pq = sc->rxqueues + kring->ring_id;
1291         bool notify;
1292
1293         notify = netmap_pt_guest_rxsync(pq->atok, pq->ktoa, kring, flags);
1294         if (notify) {
1295                 ptnet_kick(pq);
1296         }
1297
1298         return 0;
1299 }
1300
1301 static void
1302 ptnet_nm_intr(struct netmap_adapter *na, int onoff)
1303 {
1304         struct ptnet_softc *sc = if_getsoftc(na->ifp);
1305         int i;
1306
1307         for (i = 0; i < sc->num_rings; i++) {
1308                 struct ptnet_queue *pq = sc->queues + i;
1309                 pq->atok->appl_need_kick = onoff;
1310         }
1311 }
1312
1313 static void
1314 ptnet_tx_intr(void *opaque)
1315 {
1316         struct ptnet_queue *pq = opaque;
1317         struct ptnet_softc *sc = pq->sc;
1318
1319         DBG(device_printf(sc->dev, "Tx interrupt #%d\n", pq->kring_id));
1320 #ifdef PTNETMAP_STATS
1321         pq->stats.intrs ++;
1322 #endif /* PTNETMAP_STATS */
1323
1324         if (netmap_tx_irq(sc->ifp, pq->kring_id) != NM_IRQ_PASS) {
1325                 return;
1326         }
1327
1328         /* Schedule the tasqueue to flush process transmissions requests.
1329          * However, vtnet, if_em and if_igb just call ptnet_transmit() here,
1330          * at least when using MSI-X interrupts. The if_em driver, instead
1331          * schedule taskqueue when using legacy interrupts. */
1332         taskqueue_enqueue(pq->taskq, &pq->task);
1333 }
1334
1335 static void
1336 ptnet_rx_intr(void *opaque)
1337 {
1338         struct ptnet_queue *pq = opaque;
1339         struct ptnet_softc *sc = pq->sc;
1340         unsigned int unused;
1341
1342         DBG(device_printf(sc->dev, "Rx interrupt #%d\n", pq->kring_id));
1343 #ifdef PTNETMAP_STATS
1344         pq->stats.intrs ++;
1345 #endif /* PTNETMAP_STATS */
1346
1347         if (netmap_rx_irq(sc->ifp, pq->kring_id, &unused) != NM_IRQ_PASS) {
1348                 return;
1349         }
1350
1351         /* Like vtnet, if_igb and if_em drivers when using MSI-X interrupts,
1352          * receive-side processing is executed directly in the interrupt
1353          * service routine. Alternatively, we may schedule the taskqueue. */
1354         ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
1355 }
1356
1357 /* The following offloadings-related functions are taken from the vtnet
1358  * driver, but the same functionality is required for the ptnet driver.
1359  * As a temporary solution, I copied this code from vtnet and I started
1360  * to generalize it (taking away driver-specific statistic accounting),
1361  * making as little modifications as possible.
1362  * In the future we need to share these functions between vtnet and ptnet.
1363  */
1364 static int
1365 ptnet_tx_offload_ctx(struct mbuf *m, int *etype, int *proto, int *start)
1366 {
1367         struct ether_vlan_header *evh;
1368         int offset;
1369
1370         evh = mtod(m, struct ether_vlan_header *);
1371         if (evh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1372                 /* BMV: We should handle nested VLAN tags too. */
1373                 *etype = ntohs(evh->evl_proto);
1374                 offset = sizeof(struct ether_vlan_header);
1375         } else {
1376                 *etype = ntohs(evh->evl_encap_proto);
1377                 offset = sizeof(struct ether_header);
1378         }
1379
1380         switch (*etype) {
1381 #if defined(INET)
1382         case ETHERTYPE_IP: {
1383                 struct ip *ip, iphdr;
1384                 if (__predict_false(m->m_len < offset + sizeof(struct ip))) {
1385                         m_copydata(m, offset, sizeof(struct ip),
1386                             (caddr_t) &iphdr);
1387                         ip = &iphdr;
1388                 } else
1389                         ip = (struct ip *)(m->m_data + offset);
1390                 *proto = ip->ip_p;
1391                 *start = offset + (ip->ip_hl << 2);
1392                 break;
1393         }
1394 #endif
1395 #if defined(INET6)
1396         case ETHERTYPE_IPV6:
1397                 *proto = -1;
1398                 *start = ip6_lasthdr(m, offset, IPPROTO_IPV6, proto);
1399                 /* Assert the network stack sent us a valid packet. */
1400                 KASSERT(*start > offset,
1401                     ("%s: mbuf %p start %d offset %d proto %d", __func__, m,
1402                     *start, offset, *proto));
1403                 break;
1404 #endif
1405         default:
1406                 /* Here we should increment the tx_csum_bad_ethtype counter. */
1407                 return (EINVAL);
1408         }
1409
1410         return (0);
1411 }
1412
1413 static int
1414 ptnet_tx_offload_tso(if_t ifp, struct mbuf *m, int eth_type,
1415                      int offset, bool allow_ecn, struct virtio_net_hdr *hdr)
1416 {
1417         static struct timeval lastecn;
1418         static int curecn;
1419         struct tcphdr *tcp, tcphdr;
1420
1421         if (__predict_false(m->m_len < offset + sizeof(struct tcphdr))) {
1422                 m_copydata(m, offset, sizeof(struct tcphdr), (caddr_t) &tcphdr);
1423                 tcp = &tcphdr;
1424         } else
1425                 tcp = (struct tcphdr *)(m->m_data + offset);
1426
1427         hdr->hdr_len = offset + (tcp->th_off << 2);
1428         hdr->gso_size = m->m_pkthdr.tso_segsz;
1429         hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
1430             VIRTIO_NET_HDR_GSO_TCPV6;
1431
1432         if (tcp->th_flags & TH_CWR) {
1433                 /*
1434                  * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD,
1435                  * ECN support is not on a per-interface basis, but globally via
1436                  * the net.inet.tcp.ecn.enable sysctl knob. The default is off.
1437                  */
1438                 if (!allow_ecn) {
1439                         if (ppsratecheck(&lastecn, &curecn, 1))
1440                                 if_printf(ifp,
1441                                     "TSO with ECN not negotiated with host\n");
1442                         return (ENOTSUP);
1443                 }
1444                 hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1445         }
1446
1447         /* Here we should increment tx_tso counter. */
1448
1449         return (0);
1450 }
1451
1452 static struct mbuf *
1453 ptnet_tx_offload(if_t ifp, struct mbuf *m, bool allow_ecn,
1454                  struct virtio_net_hdr *hdr)
1455 {
1456         int flags, etype, csum_start, proto, error;
1457
1458         flags = m->m_pkthdr.csum_flags;
1459
1460         error = ptnet_tx_offload_ctx(m, &etype, &proto, &csum_start);
1461         if (error)
1462                 goto drop;
1463
1464         if ((etype == ETHERTYPE_IP && flags & PTNET_CSUM_OFFLOAD) ||
1465             (etype == ETHERTYPE_IPV6 && flags & PTNET_CSUM_OFFLOAD_IPV6)) {
1466                 /*
1467                  * We could compare the IP protocol vs the CSUM_ flag too,
1468                  * but that really should not be necessary.
1469                  */
1470                 hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
1471                 hdr->csum_start = csum_start;
1472                 hdr->csum_offset = m->m_pkthdr.csum_data;
1473                 /* Here we should increment the tx_csum counter. */
1474         }
1475
1476         if (flags & CSUM_TSO) {
1477                 if (__predict_false(proto != IPPROTO_TCP)) {
1478                         /* Likely failed to correctly parse the mbuf.
1479                          * Here we should increment the tx_tso_not_tcp
1480                          * counter. */
1481                         goto drop;
1482                 }
1483
1484                 KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM,
1485                     ("%s: mbuf %p TSO without checksum offload %#x",
1486                     __func__, m, flags));
1487
1488                 error = ptnet_tx_offload_tso(ifp, m, etype, csum_start,
1489                                              allow_ecn, hdr);
1490                 if (error)
1491                         goto drop;
1492         }
1493
1494         return (m);
1495
1496 drop:
1497         m_freem(m);
1498         return (NULL);
1499 }
1500
1501 static void
1502 ptnet_vlan_tag_remove(struct mbuf *m)
1503 {
1504         struct ether_vlan_header *evh;
1505
1506         evh = mtod(m, struct ether_vlan_header *);
1507         m->m_pkthdr.ether_vtag = ntohs(evh->evl_tag);
1508         m->m_flags |= M_VLANTAG;
1509
1510         /* Strip the 802.1Q header. */
1511         bcopy((char *) evh, (char *) evh + ETHER_VLAN_ENCAP_LEN,
1512             ETHER_HDR_LEN - ETHER_TYPE_LEN);
1513         m_adj(m, ETHER_VLAN_ENCAP_LEN);
1514 }
1515
1516 /*
1517  * Use the checksum offset in the VirtIO header to set the
1518  * correct CSUM_* flags.
1519  */
1520 static int
1521 ptnet_rx_csum_by_offset(struct mbuf *m, uint16_t eth_type, int ip_start,
1522                         struct virtio_net_hdr *hdr)
1523 {
1524 #if defined(INET) || defined(INET6)
1525         int offset = hdr->csum_start + hdr->csum_offset;
1526 #endif
1527
1528         /* Only do a basic sanity check on the offset. */
1529         switch (eth_type) {
1530 #if defined(INET)
1531         case ETHERTYPE_IP:
1532                 if (__predict_false(offset < ip_start + sizeof(struct ip)))
1533                         return (1);
1534                 break;
1535 #endif
1536 #if defined(INET6)
1537         case ETHERTYPE_IPV6:
1538                 if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr)))
1539                         return (1);
1540                 break;
1541 #endif
1542         default:
1543                 /* Here we should increment the rx_csum_bad_ethtype counter. */
1544                 return (1);
1545         }
1546
1547         /*
1548          * Use the offset to determine the appropriate CSUM_* flags. This is
1549          * a bit dirty, but we can get by with it since the checksum offsets
1550          * happen to be different. We assume the host host does not do IPv4
1551          * header checksum offloading.
1552          */
1553         switch (hdr->csum_offset) {
1554         case offsetof(struct udphdr, uh_sum):
1555         case offsetof(struct tcphdr, th_sum):
1556                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1557                 m->m_pkthdr.csum_data = 0xFFFF;
1558                 break;
1559         case offsetof(struct sctphdr, checksum):
1560                 m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
1561                 break;
1562         default:
1563                 /* Here we should increment the rx_csum_bad_offset counter. */
1564                 return (1);
1565         }
1566
1567         return (0);
1568 }
1569
1570 static int
1571 ptnet_rx_csum_by_parse(struct mbuf *m, uint16_t eth_type, int ip_start,
1572                        struct virtio_net_hdr *hdr)
1573 {
1574         int offset, proto;
1575
1576         switch (eth_type) {
1577 #if defined(INET)
1578         case ETHERTYPE_IP: {
1579                 struct ip *ip;
1580                 if (__predict_false(m->m_len < ip_start + sizeof(struct ip)))
1581                         return (1);
1582                 ip = (struct ip *)(m->m_data + ip_start);
1583                 proto = ip->ip_p;
1584                 offset = ip_start + (ip->ip_hl << 2);
1585                 break;
1586         }
1587 #endif
1588 #if defined(INET6)
1589         case ETHERTYPE_IPV6:
1590                 if (__predict_false(m->m_len < ip_start +
1591                     sizeof(struct ip6_hdr)))
1592                         return (1);
1593                 offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto);
1594                 if (__predict_false(offset < 0))
1595                         return (1);
1596                 break;
1597 #endif
1598         default:
1599                 /* Here we should increment the rx_csum_bad_ethtype counter. */
1600                 return (1);
1601         }
1602
1603         switch (proto) {
1604         case IPPROTO_TCP:
1605                 if (__predict_false(m->m_len < offset + sizeof(struct tcphdr)))
1606                         return (1);
1607                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1608                 m->m_pkthdr.csum_data = 0xFFFF;
1609                 break;
1610         case IPPROTO_UDP:
1611                 if (__predict_false(m->m_len < offset + sizeof(struct udphdr)))
1612                         return (1);
1613                 m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
1614                 m->m_pkthdr.csum_data = 0xFFFF;
1615                 break;
1616         case IPPROTO_SCTP:
1617                 if (__predict_false(m->m_len < offset + sizeof(struct sctphdr)))
1618                         return (1);
1619                 m->m_pkthdr.csum_flags |= CSUM_SCTP_VALID;
1620                 break;
1621         default:
1622                 /*
1623                  * For the remaining protocols, FreeBSD does not support
1624                  * checksum offloading, so the checksum will be recomputed.
1625                  */
1626 #if 0
1627                 if_printf(ifp, "cksum offload of unsupported "
1628                     "protocol eth_type=%#x proto=%d csum_start=%d "
1629                     "csum_offset=%d\n", __func__, eth_type, proto,
1630                     hdr->csum_start, hdr->csum_offset);
1631 #endif
1632                 break;
1633         }
1634
1635         return (0);
1636 }
1637
1638 /*
1639  * Set the appropriate CSUM_* flags. Unfortunately, the information
1640  * provided is not directly useful to us. The VirtIO header gives the
1641  * offset of the checksum, which is all Linux needs, but this is not
1642  * how FreeBSD does things. We are forced to peek inside the packet
1643  * a bit.
1644  *
1645  * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD
1646  * could accept the offsets and let the stack figure it out.
1647  */
1648 static int
1649 ptnet_rx_csum(struct mbuf *m, struct virtio_net_hdr *hdr)
1650 {
1651         struct ether_header *eh;
1652         struct ether_vlan_header *evh;
1653         uint16_t eth_type;
1654         int offset, error;
1655
1656         eh = mtod(m, struct ether_header *);
1657         eth_type = ntohs(eh->ether_type);
1658         if (eth_type == ETHERTYPE_VLAN) {
1659                 /* BMV: We should handle nested VLAN tags too. */
1660                 evh = mtod(m, struct ether_vlan_header *);
1661                 eth_type = ntohs(evh->evl_proto);
1662                 offset = sizeof(struct ether_vlan_header);
1663         } else
1664                 offset = sizeof(struct ether_header);
1665
1666         if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
1667                 error = ptnet_rx_csum_by_offset(m, eth_type, offset, hdr);
1668         else
1669                 error = ptnet_rx_csum_by_parse(m, eth_type, offset, hdr);
1670
1671         return (error);
1672 }
1673 /* End of offloading-related functions to be shared with vtnet. */
1674
1675 static void
1676 ptnet_ring_update(struct ptnet_queue *pq, struct netmap_kring *kring,
1677                   unsigned int head, unsigned int sync_flags)
1678 {
1679         struct netmap_ring *ring = kring->ring;
1680         struct nm_csb_atok *atok = pq->atok;
1681         struct nm_csb_ktoa *ktoa = pq->ktoa;
1682
1683         /* Some packets have been pushed to the netmap ring. We have
1684          * to tell the host to process the new packets, updating cur
1685          * and head in the CSB. */
1686         ring->head = ring->cur = head;
1687
1688         /* Mimic nm_txsync_prologue/nm_rxsync_prologue. */
1689         kring->rcur = kring->rhead = head;
1690
1691         nm_sync_kloop_appl_write(atok, kring->rcur, kring->rhead);
1692
1693         /* Kick the host if needed. */
1694         if (NM_ACCESS_ONCE(ktoa->kern_need_kick)) {
1695                 atok->sync_flags = sync_flags;
1696                 ptnet_kick(pq);
1697         }
1698 }
1699
1700 #define PTNET_TX_NOSPACE(_h, _k, _min)  \
1701         ((((_h) < (_k)->rtail) ? 0 : (_k)->nkr_num_slots) + \
1702                 (_k)->rtail - (_h)) < (_min)
1703
1704 /* This function may be called by the network stack, or by
1705  * by the taskqueue thread. */
1706 static int
1707 ptnet_drain_transmit_queue(struct ptnet_queue *pq, unsigned int budget,
1708                            bool may_resched)
1709 {
1710         struct ptnet_softc *sc = pq->sc;
1711         bool have_vnet_hdr = sc->vnet_hdr_len;
1712         struct netmap_adapter *na = &sc->ptna->dr.up;
1713         if_t ifp = sc->ifp;
1714         unsigned int batch_count = 0;
1715         struct nm_csb_atok *atok;
1716         struct nm_csb_ktoa *ktoa;
1717         struct netmap_kring *kring;
1718         struct netmap_ring *ring;
1719         struct netmap_slot *slot;
1720         unsigned int count = 0;
1721         unsigned int minspace;
1722         unsigned int head;
1723         unsigned int lim;
1724         struct mbuf *mhead;
1725         struct mbuf *mf;
1726         int nmbuf_bytes;
1727         uint8_t *nmbuf;
1728
1729         if (!PTNET_Q_TRYLOCK(pq)) {
1730                 /* We failed to acquire the lock, schedule the taskqueue. */
1731                 RD(1, "Deferring TX work");
1732                 if (may_resched) {
1733                         taskqueue_enqueue(pq->taskq, &pq->task);
1734                 }
1735
1736                 return 0;
1737         }
1738
1739         if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
1740                 PTNET_Q_UNLOCK(pq);
1741                 RD(1, "Interface is down");
1742                 return ENETDOWN;
1743         }
1744
1745         atok = pq->atok;
1746         ktoa = pq->ktoa;
1747         kring = na->tx_rings[pq->kring_id];
1748         ring = kring->ring;
1749         lim = kring->nkr_num_slots - 1;
1750         head = ring->head;
1751         minspace = sc->min_tx_space;
1752
1753         while (count < budget) {
1754                 if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1755                         /* We ran out of slot, let's see if the host has
1756                          * freed up some, by reading hwcur and hwtail from
1757                          * the CSB. */
1758                         ptnet_sync_tail(ktoa, kring);
1759
1760                         if (PTNET_TX_NOSPACE(head, kring, minspace)) {
1761                                 /* Still no slots available. Reactivate the
1762                                  * interrupts so that we can be notified
1763                                  * when some free slots are made available by
1764                                  * the host. */
1765                                 atok->appl_need_kick = 1;
1766
1767                                 /* Double check. We need a full barrier to
1768                                  * prevent the store to atok->appl_need_kick
1769                                  * to be reordered with the load from
1770                                  * ktoa->hwcur and ktoa->hwtail (store-load
1771                                  * barrier). */
1772                                 nm_stld_barrier();
1773                                 ptnet_sync_tail(ktoa, kring);
1774                                 if (likely(PTNET_TX_NOSPACE(head, kring,
1775                                                             minspace))) {
1776                                         break;
1777                                 }
1778
1779                                 RD(1, "Found more slots by doublecheck");
1780                                 /* More slots were freed before reactivating
1781                                  * the interrupts. */
1782                                 atok->appl_need_kick = 0;
1783                         }
1784                 }
1785
1786                 mhead = drbr_peek(ifp, pq->bufring);
1787                 if (!mhead) {
1788                         break;
1789                 }
1790
1791                 /* Initialize transmission state variables. */
1792                 slot = ring->slot + head;
1793                 nmbuf = NMB(na, slot);
1794                 nmbuf_bytes = 0;
1795
1796                 /* If needed, prepare the virtio-net header at the beginning
1797                  * of the first slot. */
1798                 if (have_vnet_hdr) {
1799                         struct virtio_net_hdr *vh =
1800                                         (struct virtio_net_hdr *)nmbuf;
1801
1802                         /* For performance, we could replace this memset() with
1803                          * two 8-bytes-wide writes. */
1804                         memset(nmbuf, 0, PTNET_HDR_SIZE);
1805                         if (mhead->m_pkthdr.csum_flags & PTNET_ALL_OFFLOAD) {
1806                                 mhead = ptnet_tx_offload(ifp, mhead, false,
1807                                                          vh);
1808                                 if (unlikely(!mhead)) {
1809                                         /* Packet dropped because errors
1810                                          * occurred while preparing the vnet
1811                                          * header. Let's go ahead with the next
1812                                          * packet. */
1813                                         pq->stats.errors ++;
1814                                         drbr_advance(ifp, pq->bufring);
1815                                         continue;
1816                                 }
1817                         }
1818                         ND(1, "%s: [csum_flags %lX] vnet hdr: flags %x "
1819                               "csum_start %u csum_ofs %u hdr_len = %u "
1820                               "gso_size %u gso_type %x", __func__,
1821                               mhead->m_pkthdr.csum_flags, vh->flags,
1822                               vh->csum_start, vh->csum_offset, vh->hdr_len,
1823                               vh->gso_size, vh->gso_type);
1824
1825                         nmbuf += PTNET_HDR_SIZE;
1826                         nmbuf_bytes += PTNET_HDR_SIZE;
1827                 }
1828
1829                 for (mf = mhead; mf; mf = mf->m_next) {
1830                         uint8_t *mdata = mf->m_data;
1831                         int mlen = mf->m_len;
1832
1833                         for (;;) {
1834                                 int copy = NETMAP_BUF_SIZE(na) - nmbuf_bytes;
1835
1836                                 if (mlen < copy) {
1837                                         copy = mlen;
1838                                 }
1839                                 memcpy(nmbuf, mdata, copy);
1840
1841                                 mdata += copy;
1842                                 mlen -= copy;
1843                                 nmbuf += copy;
1844                                 nmbuf_bytes += copy;
1845
1846                                 if (!mlen) {
1847                                         break;
1848                                 }
1849
1850                                 slot->len = nmbuf_bytes;
1851                                 slot->flags = NS_MOREFRAG;
1852
1853                                 head = nm_next(head, lim);
1854                                 KASSERT(head != ring->tail,
1855                                         ("Unexpectedly run out of TX space"));
1856                                 slot = ring->slot + head;
1857                                 nmbuf = NMB(na, slot);
1858                                 nmbuf_bytes = 0;
1859                         }
1860                 }
1861
1862                 /* Complete last slot and update head. */
1863                 slot->len = nmbuf_bytes;
1864                 slot->flags = 0;
1865                 head = nm_next(head, lim);
1866
1867                 /* Consume the packet just processed. */
1868                 drbr_advance(ifp, pq->bufring);
1869
1870                 /* Copy the packet to listeners. */
1871                 ETHER_BPF_MTAP(ifp, mhead);
1872
1873                 pq->stats.packets ++;
1874                 pq->stats.bytes += mhead->m_pkthdr.len;
1875                 if (mhead->m_flags & M_MCAST) {
1876                         pq->stats.mcasts ++;
1877                 }
1878
1879                 m_freem(mhead);
1880
1881                 count ++;
1882                 if (++batch_count == PTNET_TX_BATCH) {
1883                         ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1884                         batch_count = 0;
1885                 }
1886         }
1887
1888         if (batch_count) {
1889                 ptnet_ring_update(pq, kring, head, NAF_FORCE_RECLAIM);
1890         }
1891
1892         if (count >= budget && may_resched) {
1893                 DBG(RD(1, "out of budget: resched, %d mbufs pending\n",
1894                                         drbr_inuse(ifp, pq->bufring)));
1895                 taskqueue_enqueue(pq->taskq, &pq->task);
1896         }
1897
1898         PTNET_Q_UNLOCK(pq);
1899
1900         return count;
1901 }
1902
1903 static int
1904 ptnet_transmit(if_t ifp, struct mbuf *m)
1905 {
1906         struct ptnet_softc *sc = if_getsoftc(ifp);
1907         struct ptnet_queue *pq;
1908         unsigned int queue_idx;
1909         int err;
1910
1911         DBG(device_printf(sc->dev, "transmit %p\n", m));
1912
1913         /* Insert 802.1Q header if needed. */
1914         if (m->m_flags & M_VLANTAG) {
1915                 m = ether_vlanencap(m, m->m_pkthdr.ether_vtag);
1916                 if (m == NULL) {
1917                         return ENOBUFS;
1918                 }
1919                 m->m_flags &= ~M_VLANTAG;
1920         }
1921
1922         /* Get the flow-id if available. */
1923         queue_idx = (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) ?
1924                     m->m_pkthdr.flowid : curcpu;
1925
1926         if (unlikely(queue_idx >= sc->num_tx_rings)) {
1927                 queue_idx %= sc->num_tx_rings;
1928         }
1929
1930         pq = sc->queues + queue_idx;
1931
1932         err = drbr_enqueue(ifp, pq->bufring, m);
1933         if (err) {
1934                 /* ENOBUFS when the bufring is full */
1935                 RD(1, "%s: drbr_enqueue() failed %d\n",
1936                         __func__, err);
1937                 pq->stats.errors ++;
1938                 return err;
1939         }
1940
1941         if (ifp->if_capenable & IFCAP_POLLING) {
1942                 /* If polling is on, the transmit queues will be
1943                  * drained by the poller. */
1944                 return 0;
1945         }
1946
1947         err = ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
1948
1949         return (err < 0) ? err : 0;
1950 }
1951
1952 static unsigned int
1953 ptnet_rx_discard(struct netmap_kring *kring, unsigned int head)
1954 {
1955         struct netmap_ring *ring = kring->ring;
1956         struct netmap_slot *slot = ring->slot + head;
1957
1958         for (;;) {
1959                 head = nm_next(head, kring->nkr_num_slots - 1);
1960                 if (!(slot->flags & NS_MOREFRAG) || head == ring->tail) {
1961                         break;
1962                 }
1963                 slot = ring->slot + head;
1964         }
1965
1966         return head;
1967 }
1968
1969 static inline struct mbuf *
1970 ptnet_rx_slot(struct mbuf *mtail, uint8_t *nmbuf, unsigned int nmbuf_len)
1971 {
1972         uint8_t *mdata = mtod(mtail, uint8_t *) + mtail->m_len;
1973
1974         do {
1975                 unsigned int copy;
1976
1977                 if (mtail->m_len == MCLBYTES) {
1978                         struct mbuf *mf;
1979
1980                         mf = m_getcl(M_NOWAIT, MT_DATA, 0);
1981                         if (unlikely(!mf)) {
1982                                 return NULL;
1983                         }
1984
1985                         mtail->m_next = mf;
1986                         mtail = mf;
1987                         mdata = mtod(mtail, uint8_t *);
1988                         mtail->m_len = 0;
1989                 }
1990
1991                 copy = MCLBYTES - mtail->m_len;
1992                 if (nmbuf_len < copy) {
1993                         copy = nmbuf_len;
1994                 }
1995
1996                 memcpy(mdata, nmbuf, copy);
1997
1998                 nmbuf += copy;
1999                 nmbuf_len -= copy;
2000                 mdata += copy;
2001                 mtail->m_len += copy;
2002         } while (nmbuf_len);
2003
2004         return mtail;
2005 }
2006
2007 static int
2008 ptnet_rx_eof(struct ptnet_queue *pq, unsigned int budget, bool may_resched)
2009 {
2010         struct ptnet_softc *sc = pq->sc;
2011         bool have_vnet_hdr = sc->vnet_hdr_len;
2012         struct nm_csb_atok *atok = pq->atok;
2013         struct nm_csb_ktoa *ktoa = pq->ktoa;
2014         struct netmap_adapter *na = &sc->ptna->dr.up;
2015         struct netmap_kring *kring = na->rx_rings[pq->kring_id];
2016         struct netmap_ring *ring = kring->ring;
2017         unsigned int const lim = kring->nkr_num_slots - 1;
2018         unsigned int batch_count = 0;
2019         if_t ifp = sc->ifp;
2020         unsigned int count = 0;
2021         uint32_t head;
2022
2023         PTNET_Q_LOCK(pq);
2024
2025         if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
2026                 goto unlock;
2027         }
2028
2029         kring->nr_kflags &= ~NKR_PENDINTR;
2030
2031         head = ring->head;
2032         while (count < budget) {
2033                 uint32_t prev_head = head;
2034                 struct mbuf *mhead, *mtail;
2035                 struct virtio_net_hdr *vh;
2036                 struct netmap_slot *slot;
2037                 unsigned int nmbuf_len;
2038                 uint8_t *nmbuf;
2039                 int deliver = 1; /* the mbuf to the network stack. */
2040 host_sync:
2041                 if (head == ring->tail) {
2042                         /* We ran out of slot, let's see if the host has
2043                          * added some, by reading hwcur and hwtail from
2044                          * the CSB. */
2045                         ptnet_sync_tail(ktoa, kring);
2046
2047                         if (head == ring->tail) {
2048                                 /* Still no slots available. Reactivate
2049                                  * interrupts as they were disabled by the
2050                                  * host thread right before issuing the
2051                                  * last interrupt. */
2052                                 atok->appl_need_kick = 1;
2053
2054                                 /* Double check for more completed RX slots.
2055                                  * We need a full barrier to prevent the store
2056                                  * to atok->appl_need_kick to be reordered with
2057                                  * the load from ktoa->hwcur and ktoa->hwtail
2058                                  * (store-load barrier). */
2059                                 nm_stld_barrier();
2060                                 ptnet_sync_tail(ktoa, kring);
2061                                 if (likely(head == ring->tail)) {
2062                                         break;
2063                                 }
2064                                 atok->appl_need_kick = 0;
2065                         }
2066                 }
2067
2068                 /* Initialize ring state variables, possibly grabbing the
2069                  * virtio-net header. */
2070                 slot = ring->slot + head;
2071                 nmbuf = NMB(na, slot);
2072                 nmbuf_len = slot->len;
2073
2074                 vh = (struct virtio_net_hdr *)nmbuf;
2075                 if (have_vnet_hdr) {
2076                         if (unlikely(nmbuf_len < PTNET_HDR_SIZE)) {
2077                                 /* There is no good reason why host should
2078                                  * put the header in multiple netmap slots.
2079                                  * If this is the case, discard. */
2080                                 RD(1, "Fragmented vnet-hdr: dropping");
2081                                 head = ptnet_rx_discard(kring, head);
2082                                 pq->stats.iqdrops ++;
2083                                 deliver = 0;
2084                                 goto skip;
2085                         }
2086                         ND(1, "%s: vnet hdr: flags %x csum_start %u "
2087                               "csum_ofs %u hdr_len = %u gso_size %u "
2088                               "gso_type %x", __func__, vh->flags,
2089                               vh->csum_start, vh->csum_offset, vh->hdr_len,
2090                               vh->gso_size, vh->gso_type);
2091                         nmbuf += PTNET_HDR_SIZE;
2092                         nmbuf_len -= PTNET_HDR_SIZE;
2093                 }
2094
2095                 /* Allocate the head of a new mbuf chain.
2096                  * We use m_getcl() to allocate an mbuf with standard cluster
2097                  * size (MCLBYTES). In the future we could use m_getjcl()
2098                  * to choose different sizes. */
2099                 mhead = mtail = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
2100                 if (unlikely(mhead == NULL)) {
2101                         device_printf(sc->dev, "%s: failed to allocate mbuf "
2102                                       "head\n", __func__);
2103                         pq->stats.errors ++;
2104                         break;
2105                 }
2106
2107                 /* Initialize the mbuf state variables. */
2108                 mhead->m_pkthdr.len = nmbuf_len;
2109                 mtail->m_len = 0;
2110
2111                 /* Scan all the netmap slots containing the current packet. */
2112                 for (;;) {
2113                         DBG(device_printf(sc->dev, "%s: h %u t %u rcv frag "
2114                                           "len %u, flags %u\n", __func__,
2115                                           head, ring->tail, slot->len,
2116                                           slot->flags));
2117
2118                         mtail = ptnet_rx_slot(mtail, nmbuf, nmbuf_len);
2119                         if (unlikely(!mtail)) {
2120                                 /* Ouch. We ran out of memory while processing
2121                                  * a packet. We have to restore the previous
2122                                  * head position, free the mbuf chain, and
2123                                  * schedule the taskqueue to give the packet
2124                                  * another chance. */
2125                                 device_printf(sc->dev, "%s: failed to allocate"
2126                                         " mbuf frag, reset head %u --> %u\n",
2127                                         __func__, head, prev_head);
2128                                 head = prev_head;
2129                                 m_freem(mhead);
2130                                 pq->stats.errors ++;
2131                                 if (may_resched) {
2132                                         taskqueue_enqueue(pq->taskq,
2133                                                           &pq->task);
2134                                 }
2135                                 goto escape;
2136                         }
2137
2138                         /* We have to increment head irrespective of the
2139                          * NS_MOREFRAG being set or not. */
2140                         head = nm_next(head, lim);
2141
2142                         if (!(slot->flags & NS_MOREFRAG)) {
2143                                 break;
2144                         }
2145
2146                         if (unlikely(head == ring->tail)) {
2147                                 /* The very last slot prepared by the host has
2148                                  * the NS_MOREFRAG set. Drop it and continue
2149                                  * the outer cycle (to do the double-check). */
2150                                 RD(1, "Incomplete packet: dropping");
2151                                 m_freem(mhead);
2152                                 pq->stats.iqdrops ++;
2153                                 goto host_sync;
2154                         }
2155
2156                         slot = ring->slot + head;
2157                         nmbuf = NMB(na, slot);
2158                         nmbuf_len = slot->len;
2159                         mhead->m_pkthdr.len += nmbuf_len;
2160                 }
2161
2162                 mhead->m_pkthdr.rcvif = ifp;
2163                 mhead->m_pkthdr.csum_flags = 0;
2164
2165                 /* Store the queue idx in the packet header. */
2166                 mhead->m_pkthdr.flowid = pq->kring_id;
2167                 M_HASHTYPE_SET(mhead, M_HASHTYPE_OPAQUE);
2168
2169                 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
2170                         struct ether_header *eh;
2171
2172                         eh = mtod(mhead, struct ether_header *);
2173                         if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
2174                                 ptnet_vlan_tag_remove(mhead);
2175                                 /*
2176                                  * With the 802.1Q header removed, update the
2177                                  * checksum starting location accordingly.
2178                                  */
2179                                 if (vh->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
2180                                         vh->csum_start -= ETHER_VLAN_ENCAP_LEN;
2181                         }
2182                 }
2183
2184                 if (have_vnet_hdr && (vh->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM
2185                                         | VIRTIO_NET_HDR_F_DATA_VALID))) {
2186                         if (unlikely(ptnet_rx_csum(mhead, vh))) {
2187                                 m_freem(mhead);
2188                                 RD(1, "Csum offload error: dropping");
2189                                 pq->stats.iqdrops ++;
2190                                 deliver = 0;
2191                         }
2192                 }
2193
2194 skip:
2195                 count ++;
2196                 if (++batch_count >= PTNET_RX_BATCH) {
2197                         /* Some packets have been (or will be) pushed to the network
2198                          * stack. We need to update the CSB to tell the host about
2199                          * the new ring->cur and ring->head (RX buffer refill). */
2200                         ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
2201                         batch_count = 0;
2202                 }
2203
2204                 if (likely(deliver))  {
2205                         pq->stats.packets ++;
2206                         pq->stats.bytes += mhead->m_pkthdr.len;
2207
2208                         PTNET_Q_UNLOCK(pq);
2209                         (*ifp->if_input)(ifp, mhead);
2210                         PTNET_Q_LOCK(pq);
2211                         /* The ring->head index (and related indices) are
2212                          * updated under pq lock by ptnet_ring_update().
2213                          * Since we dropped the lock to call if_input(), we
2214                          * must reload ring->head and restart processing the
2215                          * ring from there. */
2216                         head = ring->head;
2217
2218                         if (unlikely(!(ifp->if_drv_flags & IFF_DRV_RUNNING))) {
2219                                 /* The interface has gone down while we didn't
2220                                  * have the lock. Stop any processing and exit. */
2221                                 goto unlock;
2222                         }
2223                 }
2224         }
2225 escape:
2226         if (batch_count) {
2227                 ptnet_ring_update(pq, kring, head, NAF_FORCE_READ);
2228
2229         }
2230
2231         if (count >= budget && may_resched) {
2232                 /* If we ran out of budget or the double-check found new
2233                  * slots to process, schedule the taskqueue. */
2234                 DBG(RD(1, "out of budget: resched h %u t %u\n",
2235                                         head, ring->tail));
2236                 taskqueue_enqueue(pq->taskq, &pq->task);
2237         }
2238 unlock:
2239         PTNET_Q_UNLOCK(pq);
2240
2241         return count;
2242 }
2243
2244 static void
2245 ptnet_rx_task(void *context, int pending)
2246 {
2247         struct ptnet_queue *pq = context;
2248
2249         DBG(RD(1, "%s: pq #%u\n", __func__, pq->kring_id));
2250         ptnet_rx_eof(pq, PTNET_RX_BUDGET, true);
2251 }
2252
2253 static void
2254 ptnet_tx_task(void *context, int pending)
2255 {
2256         struct ptnet_queue *pq = context;
2257
2258         DBG(RD(1, "%s: pq #%u\n", __func__, pq->kring_id));
2259         ptnet_drain_transmit_queue(pq, PTNET_TX_BUDGET, true);
2260 }
2261
2262 #ifdef DEVICE_POLLING
2263 /* We don't need to handle differently POLL_AND_CHECK_STATUS and
2264  * POLL_ONLY, since we don't have an Interrupt Status Register. */
2265 static int
2266 ptnet_poll(if_t ifp, enum poll_cmd cmd, int budget)
2267 {
2268         struct ptnet_softc *sc = if_getsoftc(ifp);
2269         unsigned int queue_budget;
2270         unsigned int count = 0;
2271         bool borrow = false;
2272         int i;
2273
2274         KASSERT(sc->num_rings > 0, ("Found no queues in while polling ptnet"));
2275         queue_budget = MAX(budget / sc->num_rings, 1);
2276         RD(1, "Per-queue budget is %d", queue_budget);
2277
2278         while (budget) {
2279                 unsigned int rcnt = 0;
2280
2281                 for (i = 0; i < sc->num_rings; i++) {
2282                         struct ptnet_queue *pq = sc->queues + i;
2283
2284                         if (borrow) {
2285                                 queue_budget = MIN(queue_budget, budget);
2286                                 if (queue_budget == 0) {
2287                                         break;
2288                                 }
2289                         }
2290
2291                         if (i < sc->num_tx_rings) {
2292                                 rcnt += ptnet_drain_transmit_queue(pq,
2293                                                    queue_budget, false);
2294                         } else {
2295                                 rcnt += ptnet_rx_eof(pq, queue_budget,
2296                                                       false);
2297                         }
2298                 }
2299
2300                 if (!rcnt) {
2301                         /* A scan of the queues gave no result, we can
2302                          * stop here. */
2303                         break;
2304                 }
2305
2306                 if (rcnt > budget) {
2307                         /* This may happen when initial budget < sc->num_rings,
2308                          * since one packet budget is given to each queue
2309                          * anyway. Just pretend we didn't eat "so much". */
2310                         rcnt = budget;
2311                 }
2312                 count += rcnt;
2313                 budget -= rcnt;
2314                 borrow = true;
2315         }
2316
2317
2318         return count;
2319 }
2320 #endif /* DEVICE_POLLING */