MFV r302423:

[FreeBSD/FreeBSD.git] / sys / dev / netmap / netmap.c

/*
 * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *   1. Redistributions of source code must retain the above copyright
 *      notice, this list of conditions and the following disclaimer.
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */


/*
 * $FreeBSD$
 *
 * This module supports memory mapped access to network devices,
 * see netmap(4).
 *
 * The module uses a large, memory pool allocated by the kernel
 * and accessible as mmapped memory by multiple userspace threads/processes.
 * The memory pool contains packet buffers and "netmap rings",
 * i.e. user-accessible copies of the interface's queues.
 *
 * Access to the network card works like this:
 * 1. a process/thread issues one or more open() on /dev/netmap, to create
 *    select()able file descriptor on which events are reported.
 * 2. on each descriptor, the process issues an ioctl() to identify
 *    the interface that should report events to the file descriptor.
 * 3. on each descriptor, the process issues an mmap() request to
 *    map the shared memory region within the process' address space.
 *    The list of interesting queues is indicated by a location in
 *    the shared memory region.
 * 4. using the functions in the netmap(4) userspace API, a process
 *    can look up the occupation state of a queue, access memory buffers,
 *    and retrieve received packets or enqueue packets to transmit.
 * 5. using some ioctl()s the process can synchronize the userspace view
 *    of the queue with the actual status in the kernel. This includes both
 *    receiving the notification of new packets, and transmitting new
 *    packets on the output interface.
 * 6. select() or poll() can be used to wait for events on individual
 *    transmit or receive queues (or all queues for a given interface).
 *

                SYNCHRONIZATION (USER)

The netmap rings and data structures may be shared among multiple
user threads or even independent processes.
Any synchronization among those threads/processes is delegated
to the threads themselves. Only one thread at a time can be in
a system call on the same netmap ring. The OS does not enforce
this and only guarantees against system crashes in case of
invalid usage.

                LOCKING (INTERNAL)

Within the kernel, access to the netmap rings is protected as follows:

- a spinlock on each ring, to handle producer/consumer races on
  RX rings attached to the host stack (against multiple host
  threads writing from the host stack to the same ring),
  and on 'destination' rings attached to a VALE switch
  (i.e. RX rings in VALE ports, and TX rings in NIC/host ports)
  protecting multiple active senders for the same destination)

- an atomic variable to guarantee that there is at most one
  instance of *_*xsync() on the ring at any time.
  For rings connected to user file
  descriptors, an atomic_test_and_set() protects this, and the
  lock on the ring is not actually used.
  For NIC RX rings connected to a VALE switch, an atomic_test_and_set()
  is also used to prevent multiple executions (the driver might indeed
  already guarantee this).
  For NIC TX rings connected to a VALE switch, the lock arbitrates
  access to the queue (both when allocating buffers and when pushing
  them out).

- *xsync() should be protected against initializations of the card.
  On FreeBSD most devices have the reset routine protected by
  a RING lock (ixgbe, igb, em) or core lock (re). lem is missing
  the RING protection on rx_reset(), this should be added.

  On linux there is an external lock on the tx path, which probably
  also arbitrates access to the reset routine. XXX to be revised

- a per-interface core_lock protecting access from the host stack
  while interfaces may be detached from netmap mode.
  XXX there should be no need for this lock if we detach the interfaces
  only while they are down.


--- VALE SWITCH ---

NMG_LOCK() serializes all modifications to switches and ports.
A switch cannot be deleted until all ports are gone.

For each switch, an SX lock (RWlock on linux) protects
deletion of ports. When configuring or deleting a new port, the
lock is acquired in exclusive mode (after holding NMG_LOCK).
When forwarding, the lock is acquired in shared mode (without NMG_LOCK).
The lock is held throughout the entire forwarding cycle,
during which the thread may incur in a page fault.
Hence it is important that sleepable shared locks are used.

On the rx ring, the per-port lock is grabbed initially to reserve
a number of slot in the ring, then the lock is released,
packets are copied from source to destination, and then
the lock is acquired again and the receive ring is updated.
(A similar thing is done on the tx ring for NIC and host stack
ports attached to the switch)

 */


/* --- internals ----
 *
 * Roadmap to the code that implements the above.
 *
 * > 1. a process/thread issues one or more open() on /dev/netmap, to create
 * >    select()able file descriptor on which events are reported.
 *
 *      Internally, we allocate a netmap_priv_d structure, that will be
 *      initialized on ioctl(NIOCREGIF).
 *
 *      os-specific:
 *          FreeBSD: netmap_open (netmap_freebsd.c). The priv is
 *                   per-thread.
 *          linux:   linux_netmap_open (netmap_linux.c). The priv is
 *                   per-open.
 *
 * > 2. on each descriptor, the process issues an ioctl() to identify
 * >    the interface that should report events to the file descriptor.
 *
 *      Implemented by netmap_ioctl(), NIOCREGIF case, with nmr->nr_cmd==0.
 *      Most important things happen in netmap_get_na() and
 *      netmap_do_regif(), called from there. Additional details can be
 *      found in the comments above those functions.
 *
 *      In all cases, this action creates/takes-a-reference-to a
 *      netmap_*_adapter describing the port, and allocates a netmap_if
 *      and all necessary netmap rings, filling them with netmap buffers.
 *
 *      In this phase, the sync callbacks for each ring are set (these are used
 *      in steps 5 and 6 below).  The callbacks depend on the type of adapter.
 *      The adapter creation/initialization code puts them in the
 *      netmap_adapter (fields na->nm_txsync and na->nm_rxsync).  Then, they
 *      are copied from there to the netmap_kring's during netmap_do_regif(), by
 *      the nm_krings_create() callback.  All the nm_krings_create callbacks
 *      actually call netmap_krings_create() to perform this and the other
 *      common stuff. netmap_krings_create() also takes care of the host rings,
 *      if needed, by setting their sync callbacks appropriately.
 *
 *      Additional actions depend on the kind of netmap_adapter that has been
 *      registered:
 *
 *      - netmap_hw_adapter:         [netmap.c]
 *           This is a system netdev/ifp with native netmap support.
 *           The ifp is detached from the host stack by redirecting:
 *             - transmissions (from the network stack) to netmap_transmit()
 *             - receive notifications to the nm_notify() callback for
 *               this adapter. The callback is normally netmap_notify(), unless
 *               the ifp is attached to a bridge using bwrap, in which case it
 *               is netmap_bwrap_intr_notify().
 *
 *      - netmap_generic_adapter:      [netmap_generic.c]
 *            A system netdev/ifp without native netmap support.
 *
 *      (the decision about native/non native support is taken in
 *       netmap_get_hw_na(), called by netmap_get_na())
 *
 *      - netmap_vp_adapter             [netmap_vale.c]
 *            Returned by netmap_get_bdg_na().
 *            This is a persistent or ephemeral VALE port. Ephemeral ports
 *            are created on the fly if they don't already exist, and are
 *            always attached to a bridge.
 *            Persistent VALE ports must must be created separately, and i
 *            then attached like normal NICs. The NIOCREGIF we are examining
 *            will find them only if they had previosly been created and
 *            attached (see VALE_CTL below).
 *
 *      - netmap_pipe_adapter         [netmap_pipe.c]
 *            Returned by netmap_get_pipe_na().
 *            Both pipe ends are created, if they didn't already exist.
 *
 *      - netmap_monitor_adapter      [netmap_monitor.c]
 *            Returned by netmap_get_monitor_na().
 *            If successful, the nm_sync callbacks of the monitored adapter
 *            will be intercepted by the returned monitor.
 *
 *      - netmap_bwrap_adapter        [netmap_vale.c]
 *            Cannot be obtained in this way, see VALE_CTL below
 *
 *
 *      os-specific:
 *          linux: we first go through linux_netmap_ioctl() to
 *                 adapt the FreeBSD interface to the linux one.
 *
 *
 * > 3. on each descriptor, the process issues an mmap() request to
 * >    map the shared memory region within the process' address space.
 * >    The list of interesting queues is indicated by a location in
 * >    the shared memory region.
 *
 *      os-specific:
 *          FreeBSD: netmap_mmap_single (netmap_freebsd.c).
 *          linux:   linux_netmap_mmap (netmap_linux.c).
 *
 * > 4. using the functions in the netmap(4) userspace API, a process
 * >    can look up the occupation state of a queue, access memory buffers,
 * >    and retrieve received packets or enqueue packets to transmit.
 *
 *      these actions do not involve the kernel.
 *
 * > 5. using some ioctl()s the process can synchronize the userspace view
 * >    of the queue with the actual status in the kernel. This includes both
 * >    receiving the notification of new packets, and transmitting new
 * >    packets on the output interface.
 *
 *      These are implemented in netmap_ioctl(), NIOCTXSYNC and NIOCRXSYNC
 *      cases. They invoke the nm_sync callbacks on the netmap_kring
 *      structures, as initialized in step 2 and maybe later modified
 *      by a monitor. Monitors, however, will always call the original
 *      callback before doing anything else.
 *
 *
 * > 6. select() or poll() can be used to wait for events on individual
 * >    transmit or receive queues (or all queues for a given interface).
 *
 *      Implemented in netmap_poll(). This will call the same nm_sync()
 *      callbacks as in step 5 above.
 *
 *      os-specific:
 *              linux: we first go through linux_netmap_poll() to adapt
 *                     the FreeBSD interface to the linux one.
 *
 *
 *  ----  VALE_CTL -----
 *
 *  VALE switches are controlled by issuing a NIOCREGIF with a non-null
 *  nr_cmd in the nmreq structure. These subcommands are handled by
 *  netmap_bdg_ctl() in netmap_vale.c. Persistent VALE ports are created
 *  and destroyed by issuing the NETMAP_BDG_NEWIF and NETMAP_BDG_DELIF
 *  subcommands, respectively.
 *
 *  Any network interface known to the system (including a persistent VALE
 *  port) can be attached to a VALE switch by issuing the
 *  NETMAP_BDG_ATTACH subcommand. After the attachment, persistent VALE ports
 *  look exactly like ephemeral VALE ports (as created in step 2 above).  The
 *  attachment of other interfaces, instead, requires the creation of a
 *  netmap_bwrap_adapter.  Moreover, the attached interface must be put in
 *  netmap mode. This may require the creation of a netmap_generic_adapter if
 *  we have no native support for the interface, or if generic adapters have
 *  been forced by sysctl.
 *
 *  Both persistent VALE ports and bwraps are handled by netmap_get_bdg_na(),
 *  called by nm_bdg_ctl_attach(), and discriminated by the nm_bdg_attach()
 *  callback.  In the case of the bwrap, the callback creates the
 *  netmap_bwrap_adapter.  The initialization of the bwrap is then
 *  completed by calling netmap_do_regif() on it, in the nm_bdg_ctl()
 *  callback (netmap_bwrap_bdg_ctl in netmap_vale.c).
 *  A generic adapter for the wrapped ifp will be created if needed, when
 *  netmap_get_bdg_na() calls netmap_get_hw_na().
 *
 *
 *  ---- DATAPATHS -----
 *
 *              -= SYSTEM DEVICE WITH NATIVE SUPPORT =-
 *
 *    na == NA(ifp) == netmap_hw_adapter created in DEVICE_netmap_attach()
 *
 *    - tx from netmap userspace:
 *       concurrently:
 *           1) ioctl(NIOCTXSYNC)/netmap_poll() in process context
 *                kring->nm_sync() == DEVICE_netmap_txsync()
 *           2) device interrupt handler
 *                na->nm_notify()  == netmap_notify()
 *    - rx from netmap userspace:
 *       concurrently:
 *           1) ioctl(NIOCRXSYNC)/netmap_poll() in process context
 *                kring->nm_sync() == DEVICE_netmap_rxsync()
 *           2) device interrupt handler
 *                na->nm_notify()  == netmap_notify()
 *    - rx from host stack
 *       concurrently:
 *           1) host stack
 *                netmap_transmit()
 *                  na->nm_notify  == netmap_notify()
 *           2) ioctl(NIOCRXSYNC)/netmap_poll() in process context
 *                kring->nm_sync() == netmap_rxsync_from_host_compat
 *                  netmap_rxsync_from_host(na, NULL, NULL)
 *    - tx to host stack
 *           ioctl(NIOCTXSYNC)/netmap_poll() in process context
 *             kring->nm_sync() == netmap_txsync_to_host_compat
 *               netmap_txsync_to_host(na)
 *                 NM_SEND_UP()
 *                   FreeBSD: na->if_input() == ?? XXX
 *                   linux: netif_rx() with NM_MAGIC_PRIORITY_RX
 *
 *
 *
 *               -= SYSTEM DEVICE WITH GENERIC SUPPORT =-
 *
 *    na == NA(ifp) == generic_netmap_adapter created in generic_netmap_attach()
 *
 *    - tx from netmap userspace:
 *       concurrently:
 *           1) ioctl(NIOCTXSYNC)/netmap_poll() in process context
 *               kring->nm_sync() == generic_netmap_txsync()
 *                   linux:   dev_queue_xmit() with NM_MAGIC_PRIORITY_TX
 *                       generic_ndo_start_xmit()
 *                           orig. dev. start_xmit
 *                   FreeBSD: na->if_transmit() == orig. dev if_transmit
 *           2) generic_mbuf_destructor()
 *                   na->nm_notify() == netmap_notify()
 *    - rx from netmap userspace:
 *           1) ioctl(NIOCRXSYNC)/netmap_poll() in process context
 *               kring->nm_sync() == generic_netmap_rxsync()
 *                   mbq_safe_dequeue()
 *           2) device driver
 *               generic_rx_handler()
 *                   mbq_safe_enqueue()
 *                   na->nm_notify() == netmap_notify()
 *    - rx from host stack:
 *        concurrently:
 *           1) host stack
 *               linux: generic_ndo_start_xmit()
 *                   netmap_transmit()
 *               FreeBSD: ifp->if_input() == netmap_transmit
 *               both:
 *                       na->nm_notify() == netmap_notify()
 *           2) ioctl(NIOCRXSYNC)/netmap_poll() in process context
 *                kring->nm_sync() == netmap_rxsync_from_host_compat
 *                  netmap_rxsync_from_host(na, NULL, NULL)
 *    - tx to host stack:
 *           ioctl(NIOCTXSYNC)/netmap_poll() in process context
 *             kring->nm_sync() == netmap_txsync_to_host_compat
 *               netmap_txsync_to_host(na)
 *                 NM_SEND_UP()
 *                   FreeBSD: na->if_input() == ??? XXX
 *                   linux: netif_rx() with NM_MAGIC_PRIORITY_RX
 *
 *
 *                           -= VALE =-
 *
 *   INCOMING:
 *
 *      - VALE ports:
 *          ioctl(NIOCTXSYNC)/netmap_poll() in process context
 *              kring->nm_sync() == netmap_vp_txsync()
 *
 *      - system device with native support:
 *         from cable:
 *             interrupt
 *                na->nm_notify() == netmap_bwrap_intr_notify(ring_nr != host ring)
 *                     kring->nm_sync() == DEVICE_netmap_rxsync()
 *                     netmap_vp_txsync()
 *                     kring->nm_sync() == DEVICE_netmap_rxsync()
 *         from host stack:
 *             netmap_transmit()
 *                na->nm_notify() == netmap_bwrap_intr_notify(ring_nr == host ring)
 *                     kring->nm_sync() == netmap_rxsync_from_host_compat()
 *                     netmap_vp_txsync()
 *
 *      - system device with generic support:
 *         from device driver:
 *            generic_rx_handler()
 *                na->nm_notify() == netmap_bwrap_intr_notify(ring_nr != host ring)
 *                     kring->nm_sync() == generic_netmap_rxsync()
 *                     netmap_vp_txsync()
 *                     kring->nm_sync() == generic_netmap_rxsync()
 *         from host stack:
 *            netmap_transmit()
 *                na->nm_notify() == netmap_bwrap_intr_notify(ring_nr == host ring)
 *                     kring->nm_sync() == netmap_rxsync_from_host_compat()
 *                     netmap_vp_txsync()
 *
 *   (all cases) --> nm_bdg_flush()
 *                      dest_na->nm_notify() == (see below)
 *
 *   OUTGOING:
 *
 *      - VALE ports:
 *         concurrently:
 *             1) ioctlNIOCRXSYNC)/netmap_poll() in process context
 *                    kring->nm_sync() == netmap_vp_rxsync()
 *             2) from nm_bdg_flush()
 *                    na->nm_notify() == netmap_notify()
 *
 *      - system device with native support:
 *          to cable:
 *             na->nm_notify() == netmap_bwrap_notify()
 *                 netmap_vp_rxsync()
 *                 kring->nm_sync() == DEVICE_netmap_txsync()
 *                 netmap_vp_rxsync()
 *          to host stack:
 *                 netmap_vp_rxsync()
 *                 kring->nm_sync() == netmap_txsync_to_host_compat
 *                 netmap_vp_rxsync_locked()
 *
 *      - system device with generic adapter:
 *          to device driver:
 *             na->nm_notify() == netmap_bwrap_notify()
 *                 netmap_vp_rxsync()
 *                 kring->nm_sync() == generic_netmap_txsync()
 *                 netmap_vp_rxsync()
 *          to host stack:
 *                 netmap_vp_rxsync()
 *                 kring->nm_sync() == netmap_txsync_to_host_compat
 *                 netmap_vp_rxsync()
 *
 */

/*
 * OS-specific code that is used only within this file.
 * Other OS-specific code that must be accessed by drivers
 * is present in netmap_kern.h
 */

#if defined(__FreeBSD__)
#include <sys/cdefs.h> /* prerequisite */
#include <sys/types.h>
#include <sys/errno.h>
#include <sys/param.h>  /* defines used in kernel.h */
#include <sys/kernel.h> /* types used in module initialization */
#include <sys/conf.h>   /* cdevsw struct, UID, GID */
#include <sys/filio.h>  /* FIONBIO */
#include <sys/sockio.h>
#include <sys/socketvar.h>      /* struct socket */
#include <sys/malloc.h>
#include <sys/poll.h>
#include <sys/rwlock.h>
#include <sys/socket.h> /* sockaddrs */
#include <sys/selinfo.h>
#include <sys/sysctl.h>
#include <sys/jail.h>
#include <net/vnet.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/bpf.h>            /* BIOCIMMEDIATE */
#include <machine/bus.h>        /* bus_dmamap_* */
#include <sys/endian.h>
#include <sys/refcount.h>


/* reduce conditional code */
// linux API, use for the knlist in FreeBSD
/* use a private mutex for the knlist */
#define init_waitqueue_head(x) do {                     \
        struct mtx *m = &(x)->m;                        \
        mtx_init(m, "nm_kn_lock", NULL, MTX_DEF);       \
        knlist_init_mtx(&(x)->si.si_note, m);           \
    } while (0)

#define OS_selrecord(a, b)      selrecord(a, &((b)->si))
#define OS_selwakeup(a, b)      freebsd_selwakeup(a, b)

#elif defined(linux)

#include "bsd_glue.h"



#elif defined(__APPLE__)

#warning OSX support is only partial
#include "osx_glue.h"

#else

#error  Unsupported platform

#endif /* unsupported */

/*
 * common headers
 */
#include <net/netmap.h>
#include <dev/netmap/netmap_kern.h>
#include <dev/netmap/netmap_mem2.h>


MALLOC_DEFINE(M_NETMAP, "netmap", "Network memory map");

/* user-controlled variables */
int netmap_verbose;

static int netmap_no_timestamp; /* don't timestamp on rxsync */

SYSCTL_NODE(_dev, OID_AUTO, netmap, CTLFLAG_RW, 0, "Netmap args");
SYSCTL_INT(_dev_netmap, OID_AUTO, verbose,
    CTLFLAG_RW, &netmap_verbose, 0, "Verbose mode");
SYSCTL_INT(_dev_netmap, OID_AUTO, no_timestamp,
    CTLFLAG_RW, &netmap_no_timestamp, 0, "no_timestamp");
int netmap_mitigate = 1;
SYSCTL_INT(_dev_netmap, OID_AUTO, mitigate, CTLFLAG_RW, &netmap_mitigate, 0, "");
int netmap_no_pendintr = 1;
SYSCTL_INT(_dev_netmap, OID_AUTO, no_pendintr,
    CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets.");
int netmap_txsync_retry = 2;
SYSCTL_INT(_dev_netmap, OID_AUTO, txsync_retry, CTLFLAG_RW,
    &netmap_txsync_retry, 0 , "Number of txsync loops in bridge's flush.");

int netmap_adaptive_io = 0;
SYSCTL_INT(_dev_netmap, OID_AUTO, adaptive_io, CTLFLAG_RW,
    &netmap_adaptive_io, 0 , "Adaptive I/O on paravirt");

int netmap_flags = 0;   /* debug flags */
int netmap_fwd = 0;     /* force transparent mode */

/*
 * netmap_admode selects the netmap mode to use.
 * Invalid values are reset to NETMAP_ADMODE_BEST
 */
enum { NETMAP_ADMODE_BEST = 0,  /* use native, fallback to generic */
        NETMAP_ADMODE_NATIVE,   /* either native or none */
        NETMAP_ADMODE_GENERIC,  /* force generic */
        NETMAP_ADMODE_LAST };
static int netmap_admode = NETMAP_ADMODE_BEST;

int netmap_generic_mit = 100*1000;   /* Generic mitigation interval in nanoseconds. */
int netmap_generic_ringsize = 1024;   /* Generic ringsize. */
int netmap_generic_rings = 1;   /* number of queues in generic. */

SYSCTL_INT(_dev_netmap, OID_AUTO, flags, CTLFLAG_RW, &netmap_flags, 0 , "");
SYSCTL_INT(_dev_netmap, OID_AUTO, fwd, CTLFLAG_RW, &netmap_fwd, 0 , "");
SYSCTL_INT(_dev_netmap, OID_AUTO, admode, CTLFLAG_RW, &netmap_admode, 0 , "");
SYSCTL_INT(_dev_netmap, OID_AUTO, generic_mit, CTLFLAG_RW, &netmap_generic_mit, 0 , "");
SYSCTL_INT(_dev_netmap, OID_AUTO, generic_ringsize, CTLFLAG_RW, &netmap_generic_ringsize, 0 , "");
SYSCTL_INT(_dev_netmap, OID_AUTO, generic_rings, CTLFLAG_RW, &netmap_generic_rings, 0 , "");

NMG_LOCK_T      netmap_global_lock;
int netmap_use_count = 0; /* number of active netmap instances */

/*
 * mark the ring as stopped, and run through the locks
 * to make sure other users get to see it.
 */
static void
netmap_disable_ring(struct netmap_kring *kr)
{
        kr->nkr_stopped = 1;
        nm_kr_get(kr);
        mtx_lock(&kr->q_lock);
        mtx_unlock(&kr->q_lock);
        nm_kr_put(kr);
}

/* stop or enable a single ring */
void
netmap_set_ring(struct netmap_adapter *na, u_int ring_id, enum txrx t, int stopped)
{
        if (stopped)
                netmap_disable_ring(NMR(na, t) + ring_id);
        else
                NMR(na, t)[ring_id].nkr_stopped = 0;
}


/* stop or enable all the rings of na */
void
netmap_set_all_rings(struct netmap_adapter *na, int stopped)
{
        int i;
        enum txrx t;

        if (!nm_netmap_on(na))
                return;

        for_rx_tx(t) {
                for (i = 0; i < netmap_real_rings(na, t); i++) {
                        netmap_set_ring(na, i, t, stopped);
                }
        }
}

/*
 * Convenience function used in drivers.  Waits for current txsync()s/rxsync()s
 * to finish and prevents any new one from starting.  Call this before turning
 * netmap mode off, or before removing the hardware rings (e.g., on module
 * onload).  As a rule of thumb for linux drivers, this should be placed near
 * each napi_disable().
 */
void
netmap_disable_all_rings(struct ifnet *ifp)
{
        netmap_set_all_rings(NA(ifp), 1 /* stopped */);
}

/*
 * Convenience function used in drivers.  Re-enables rxsync and txsync on the
 * adapter's rings In linux drivers, this should be placed near each
 * napi_enable().
 */
void
netmap_enable_all_rings(struct ifnet *ifp)
{
        netmap_set_all_rings(NA(ifp), 0 /* enabled */);
}


/*
 * generic bound_checking function
 */
u_int
nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg)
{
        u_int oldv = *v;
        const char *op = NULL;

        if (dflt < lo)
                dflt = lo;
        if (dflt > hi)
                dflt = hi;
        if (oldv < lo) {
                *v = dflt;
                op = "Bump";
        } else if (oldv > hi) {
                *v = hi;
                op = "Clamp";
        }
        if (op && msg)
                printf("%s %s to %d (was %d)\n", op, msg, *v, oldv);
        return *v;
}


/*
 * packet-dump function, user-supplied or static buffer.
 * The destination buffer must be at least 30+4*len
 */
const char *
nm_dump_buf(char *p, int len, int lim, char *dst)
{
        static char _dst[8192];
        int i, j, i0;
        static char hex[] ="0123456789abcdef";
        char *o;        /* output position */

#define P_HI(x) hex[((x) & 0xf0)>>4]
#define P_LO(x) hex[((x) & 0xf)]
#define P_C(x)  ((x) >= 0x20 && (x) <= 0x7e ? (x) : '.')
        if (!dst)
                dst = _dst;
        if (lim <= 0 || lim > len)
                lim = len;
        o = dst;
        sprintf(o, "buf 0x%p len %d lim %d\n", p, len, lim);
        o += strlen(o);
        /* hexdump routine */
        for (i = 0; i < lim; ) {
                sprintf(o, "%5d: ", i);
                o += strlen(o);
                memset(o, ' ', 48);
                i0 = i;
                for (j=0; j < 16 && i < lim; i++, j++) {
                        o[j*3] = P_HI(p[i]);
                        o[j*3+1] = P_LO(p[i]);
                }
                i = i0;
                for (j=0; j < 16 && i < lim; i++, j++)
                        o[j + 48] = P_C(p[i]);
                o[j+48] = '\n';
                o += j+49;
        }
        *o = '\0';
#undef P_HI
#undef P_LO
#undef P_C
        return dst;
}


/*
 * Fetch configuration from the device, to cope with dynamic
 * reconfigurations after loading the module.
 */
/* call with NMG_LOCK held */
int
netmap_update_config(struct netmap_adapter *na)
{
        u_int txr, txd, rxr, rxd;

        txr = txd = rxr = rxd = 0;
        if (na->nm_config == NULL ||
            na->nm_config(na, &txr, &txd, &rxr, &rxd))
        {
                /* take whatever we had at init time */
                txr = na->num_tx_rings;
                txd = na->num_tx_desc;
                rxr = na->num_rx_rings;
                rxd = na->num_rx_desc;
        }

        if (na->num_tx_rings == txr && na->num_tx_desc == txd &&
            na->num_rx_rings == rxr && na->num_rx_desc == rxd)
                return 0; /* nothing changed */
        if (netmap_verbose || na->active_fds > 0) {
                D("stored config %s: txring %d x %d, rxring %d x %d",
                        na->name,
                        na->num_tx_rings, na->num_tx_desc,
                        na->num_rx_rings, na->num_rx_desc);
                D("new config %s: txring %d x %d, rxring %d x %d",
                        na->name, txr, txd, rxr, rxd);
        }
        if (na->active_fds == 0) {
                D("configuration changed (but fine)");
                na->num_tx_rings = txr;
                na->num_tx_desc = txd;
                na->num_rx_rings = rxr;
                na->num_rx_desc = rxd;
                return 0;
        }
        D("configuration changed while active, this is bad...");
        return 1;
}

static void netmap_txsync_to_host(struct netmap_adapter *na);
static int netmap_rxsync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait);

/* kring->nm_sync callback for the host tx ring */
static int
netmap_txsync_to_host_compat(struct netmap_kring *kring, int flags)
{
        (void)flags; /* unused */
        netmap_txsync_to_host(kring->na);
        return 0;
}

/* kring->nm_sync callback for the host rx ring */
static int
netmap_rxsync_from_host_compat(struct netmap_kring *kring, int flags)
{
        (void)flags; /* unused */
        netmap_rxsync_from_host(kring->na, NULL, NULL);
        return 0;
}



/* create the krings array and initialize the fields common to all adapters.
 * The array layout is this:
 *
 *                    +----------+
 * na->tx_rings ----->|          | \
 *                    |          |  } na->num_tx_ring
 *                    |          | /
 *                    +----------+
 *                    |          |    host tx kring
 * na->rx_rings ----> +----------+
 *                    |          | \
 *                    |          |  } na->num_rx_rings
 *                    |          | /
 *                    +----------+
 *                    |          |    host rx kring
 *                    +----------+
 * na->tailroom ----->|          | \
 *                    |          |  } tailroom bytes
 *                    |          | /
 *                    +----------+
 *
 * Note: for compatibility, host krings are created even when not needed.
 * The tailroom space is currently used by vale ports for allocating leases.
 */
/* call with NMG_LOCK held */
int
netmap_krings_create(struct netmap_adapter *na, u_int tailroom)
{
        u_int i, len, ndesc;
        struct netmap_kring *kring;
        u_int n[NR_TXRX];
        enum txrx t;

        /* account for the (possibly fake) host rings */
        n[NR_TX] = na->num_tx_rings + 1;
        n[NR_RX] = na->num_rx_rings + 1;

        len = (n[NR_TX] + n[NR_RX]) * sizeof(struct netmap_kring) + tailroom;

        na->tx_rings = malloc((size_t)len, M_DEVBUF, M_NOWAIT | M_ZERO);
        if (na->tx_rings == NULL) {
                D("Cannot allocate krings");
                return ENOMEM;
        }
        na->rx_rings = na->tx_rings + n[NR_TX];

        /*
         * All fields in krings are 0 except the one initialized below.
         * but better be explicit on important kring fields.
         */
        for_rx_tx(t) {
                ndesc = nma_get_ndesc(na, t);
                for (i = 0; i < n[t]; i++) {
                        kring = &NMR(na, t)[i];
                        bzero(kring, sizeof(*kring));
                        kring->na = na;
                        kring->ring_id = i;
                        kring->tx = t;
                        kring->nkr_num_slots = ndesc;
                        if (i < nma_get_nrings(na, t)) {
                                kring->nm_sync = (t == NR_TX ? na->nm_txsync : na->nm_rxsync);
                        } else if (i == na->num_tx_rings) {
                                kring->nm_sync = (t == NR_TX ?
                                                netmap_txsync_to_host_compat :
                                                netmap_rxsync_from_host_compat);
                        }
                        kring->nm_notify = na->nm_notify;
                        kring->rhead = kring->rcur = kring->nr_hwcur = 0;
                        /*
                         * IMPORTANT: Always keep one slot empty.
                         */
                        kring->rtail = kring->nr_hwtail = (t == NR_TX ? ndesc - 1 : 0);
                        snprintf(kring->name, sizeof(kring->name) - 1, "%s %s%d", na->name, 
                                        nm_txrx2str(t), i);
                        ND("ktx %s h %d c %d t %d",
                                kring->name, kring->rhead, kring->rcur, kring->rtail);
                        mtx_init(&kring->q_lock, (t == NR_TX ? "nm_txq_lock" : "nm_rxq_lock"), NULL, MTX_DEF);
                        init_waitqueue_head(&kring->si);
                }
                init_waitqueue_head(&na->si[t]);
        }

        na->tailroom = na->rx_rings + n[NR_RX];

        return 0;
}


#ifdef __FreeBSD__
static void
netmap_knlist_destroy(NM_SELINFO_T *si)
{
        /* XXX kqueue(9) needed; these will mirror knlist_init. */
        knlist_delete(&si->si.si_note, curthread, 0 /* not locked */ );
        knlist_destroy(&si->si.si_note);
        /* now we don't need the mutex anymore */
        mtx_destroy(&si->m);
}
#endif /* __FreeBSD__ */


/* undo the actions performed by netmap_krings_create */
/* call with NMG_LOCK held */
void
netmap_krings_delete(struct netmap_adapter *na)
{
        struct netmap_kring *kring = na->tx_rings;
        enum txrx t;

        for_rx_tx(t)
                netmap_knlist_destroy(&na->si[t]);

        /* we rely on the krings layout described above */
        for ( ; kring != na->tailroom; kring++) {
                mtx_destroy(&kring->q_lock);
                netmap_knlist_destroy(&kring->si);
        }
        free(na->tx_rings, M_DEVBUF);
        na->tx_rings = na->rx_rings = na->tailroom = NULL;
}


/*
 * Destructor for NIC ports. They also have an mbuf queue
 * on the rings connected to the host so we need to purge
 * them first.
 */
/* call with NMG_LOCK held */
static void
netmap_hw_krings_delete(struct netmap_adapter *na)
{
        struct mbq *q = &na->rx_rings[na->num_rx_rings].rx_queue;

        ND("destroy sw mbq with len %d", mbq_len(q));
        mbq_purge(q);
        mbq_safe_destroy(q);
        netmap_krings_delete(na);
}



/*
 * Undo everything that was done in netmap_do_regif(). In particular,
 * call nm_register(ifp,0) to stop netmap mode on the interface and
 * revert to normal operation.
 */
/* call with NMG_LOCK held */
static void netmap_unset_ringid(struct netmap_priv_d *);
static void netmap_rel_exclusive(struct netmap_priv_d *);
static void
netmap_do_unregif(struct netmap_priv_d *priv)
{
        struct netmap_adapter *na = priv->np_na;

        NMG_LOCK_ASSERT();
        na->active_fds--;
        /* release exclusive use if it was requested on regif */
        netmap_rel_exclusive(priv);
        if (na->active_fds <= 0) {      /* last instance */

                if (netmap_verbose)
                        D("deleting last instance for %s", na->name);

#ifdef  WITH_MONITOR
                /* walk through all the rings and tell any monitor
                 * that the port is going to exit netmap mode
                 */
                netmap_monitor_stop(na);
#endif
                /*
                 * (TO CHECK) This function is only called
                 * when the last reference to this file descriptor goes
                 * away. This means we cannot have any pending poll()
                 * or interrupt routine operating on the structure.
                 * XXX The file may be closed in a thread while
                 * another thread is using it.
                 * Linux keeps the file opened until the last reference
                 * by any outstanding ioctl/poll or mmap is gone.
                 * FreeBSD does not track mmap()s (but we do) and
                 * wakes up any sleeping poll(). Need to check what
                 * happens if the close() occurs while a concurrent
                 * syscall is running.
                 */
                na->nm_register(na, 0); /* off, clear flags */
                /* Wake up any sleeping threads. netmap_poll will
                 * then return POLLERR
                 * XXX The wake up now must happen during *_down(), when
                 * we order all activities to stop. -gl
                 */
                /* delete rings and buffers */
                netmap_mem_rings_delete(na);
                na->nm_krings_delete(na);
        }
        /* possibily decrement counter of tx_si/rx_si users */
        netmap_unset_ringid(priv);
        /* delete the nifp */
        netmap_mem_if_delete(na, priv->np_nifp);
        /* drop the allocator */
        netmap_mem_deref(na->nm_mem, na);
        /* mark the priv as unregistered */
        priv->np_na = NULL;
        priv->np_nifp = NULL;
}

/* call with NMG_LOCK held */
static __inline int
nm_si_user(struct netmap_priv_d *priv, enum txrx t)
{
        return (priv->np_na != NULL &&
                (priv->np_qlast[t] - priv->np_qfirst[t] > 1));
}

/*
 * Destructor of the netmap_priv_d, called when the fd is closed
 * Action: undo all the things done by NIOCREGIF,
 * On FreeBSD we need to track whether there are active mmap()s,
 * and we use np_active_mmaps for that. On linux, the field is always 0.
 * Return: 1 if we can free priv, 0 otherwise.
 *
 */
/* call with NMG_LOCK held */
int
netmap_dtor_locked(struct netmap_priv_d *priv)
{
        struct netmap_adapter *na = priv->np_na;

        /* number of active references to this fd */
        if (--priv->np_refs > 0) {
                return 0;
        }
        netmap_use_count--;
        if (!na) {
                return 1; //XXX is it correct?
        }
        netmap_do_unregif(priv);
        netmap_adapter_put(na);
        return 1;
}


/* call with NMG_LOCK *not* held */
void
netmap_dtor(void *data)
{
        struct netmap_priv_d *priv = data;
        int last_instance;

        NMG_LOCK();
        last_instance = netmap_dtor_locked(priv);
        NMG_UNLOCK();
        if (last_instance) {
                bzero(priv, sizeof(*priv));     /* for safety */
                free(priv, M_DEVBUF);
        }
}




/*
 * Handlers for synchronization of the queues from/to the host.
 * Netmap has two operating modes:
 * - in the default mode, the rings connected to the host stack are
 *   just another ring pair managed by userspace;
 * - in transparent mode (XXX to be defined) incoming packets
 *   (from the host or the NIC) are marked as NS_FORWARD upon
 *   arrival, and the user application has a chance to reset the
 *   flag for packets that should be dropped.
 *   On the RXSYNC or poll(), packets in RX rings between
 *   kring->nr_kcur and ring->cur with NS_FORWARD still set are moved
 *   to the other side.
 * The transfer NIC --> host is relatively easy, just encapsulate
 * into mbufs and we are done. The host --> NIC side is slightly
 * harder because there might not be room in the tx ring so it
 * might take a while before releasing the buffer.
 */


/*
 * pass a chain of buffers to the host stack as coming from 'dst'
 * We do not need to lock because the queue is private.
 */
static void
netmap_send_up(struct ifnet *dst, struct mbq *q)
{
        struct mbuf *m;

        /* send packets up, outside the lock */
        while ((m = mbq_dequeue(q)) != NULL) {
                if (netmap_verbose & NM_VERB_HOST)
                        D("sending up pkt %p size %d", m, MBUF_LEN(m));
                NM_SEND_UP(dst, m);
        }
        mbq_destroy(q);
}


/*
 * put a copy of the buffers marked NS_FORWARD into an mbuf chain.
 * Take packets from hwcur to ring->head marked NS_FORWARD (or forced)
 * and pass them up. Drop remaining packets in the unlikely event
 * of an mbuf shortage.
 */
static void
netmap_grab_packets(struct netmap_kring *kring, struct mbq *q, int force)
{
        u_int const lim = kring->nkr_num_slots - 1;
        u_int const head = kring->rhead;
        u_int n;
        struct netmap_adapter *na = kring->na;

        for (n = kring->nr_hwcur; n != head; n = nm_next(n, lim)) {
                struct mbuf *m;
                struct netmap_slot *slot = &kring->ring->slot[n];

                if ((slot->flags & NS_FORWARD) == 0 && !force)
                        continue;
                if (slot->len < 14 || slot->len > NETMAP_BUF_SIZE(na)) {
                        RD(5, "bad pkt at %d len %d", n, slot->len);
                        continue;
                }
                slot->flags &= ~NS_FORWARD; // XXX needed ?
                /* XXX TODO: adapt to the case of a multisegment packet */
                m = m_devget(NMB(na, slot), slot->len, 0, na->ifp, NULL);

                if (m == NULL)
                        break;
                mbq_enqueue(q, m);
        }
}


/*
 * Send to the NIC rings packets marked NS_FORWARD between
 * kring->nr_hwcur and kring->rhead
 * Called under kring->rx_queue.lock on the sw rx ring,
 */
static u_int
netmap_sw_to_nic(struct netmap_adapter *na)
{
        struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
        struct netmap_slot *rxslot = kring->ring->slot;
        u_int i, rxcur = kring->nr_hwcur;
        u_int const head = kring->rhead;
        u_int const src_lim = kring->nkr_num_slots - 1;
        u_int sent = 0;

        /* scan rings to find space, then fill as much as possible */
        for (i = 0; i < na->num_tx_rings; i++) {
                struct netmap_kring *kdst = &na->tx_rings[i];
                struct netmap_ring *rdst = kdst->ring;
                u_int const dst_lim = kdst->nkr_num_slots - 1;

                /* XXX do we trust ring or kring->rcur,rtail ? */
                for (; rxcur != head && !nm_ring_empty(rdst);
                     rxcur = nm_next(rxcur, src_lim) ) {
                        struct netmap_slot *src, *dst, tmp;
                        u_int dst_cur = rdst->cur;

                        src = &rxslot[rxcur];
                        if ((src->flags & NS_FORWARD) == 0 && !netmap_fwd)
                                continue;

                        sent++;

                        dst = &rdst->slot[dst_cur];

                        tmp = *src;

                        src->buf_idx = dst->buf_idx;
                        src->flags = NS_BUF_CHANGED;

                        dst->buf_idx = tmp.buf_idx;
                        dst->len = tmp.len;
                        dst->flags = NS_BUF_CHANGED;

                        rdst->cur = nm_next(dst_cur, dst_lim);
                }
                /* if (sent) XXX txsync ? */
        }
        return sent;
}


/*
 * netmap_txsync_to_host() passes packets up. We are called from a
 * system call in user process context, and the only contention
 * can be among multiple user threads erroneously calling
 * this routine concurrently.
 */
static void
netmap_txsync_to_host(struct netmap_adapter *na)
{
        struct netmap_kring *kring = &na->tx_rings[na->num_tx_rings];
        u_int const lim = kring->nkr_num_slots - 1;
        u_int const head = kring->rhead;
        struct mbq q;

        /* Take packets from hwcur to head and pass them up.
         * force head = cur since netmap_grab_packets() stops at head
         * In case of no buffers we give up. At the end of the loop,
         * the queue is drained in all cases.
         */
        mbq_init(&q);
        netmap_grab_packets(kring, &q, 1 /* force */);
        ND("have %d pkts in queue", mbq_len(&q));
        kring->nr_hwcur = head;
        kring->nr_hwtail = head + lim;
        if (kring->nr_hwtail > lim)
                kring->nr_hwtail -= lim + 1;

        netmap_send_up(na->ifp, &q);
}


/*
 * rxsync backend for packets coming from the host stack.
 * They have been put in kring->rx_queue by netmap_transmit().
 * We protect access to the kring using kring->rx_queue.lock
 *
 * This routine also does the selrecord if called from the poll handler
 * (we know because td != NULL).
 *
 * NOTE: on linux, selrecord() is defined as a macro and uses pwait
 *     as an additional hidden argument.
 * returns the number of packets delivered to tx queues in
 * transparent mode, or a negative value if error
 */
static int
netmap_rxsync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait)
{
        struct netmap_kring *kring = &na->rx_rings[na->num_rx_rings];
        struct netmap_ring *ring = kring->ring;
        u_int nm_i, n;
        u_int const lim = kring->nkr_num_slots - 1;
        u_int const head = kring->rhead;
        int ret = 0;
        struct mbq *q = &kring->rx_queue, fq;

        (void)pwait;    /* disable unused warnings */
        (void)td;

        mbq_init(&fq); /* fq holds packets to be freed */

        mbq_lock(q);

        /* First part: import newly received packets */
        n = mbq_len(q);
        if (n) { /* grab packets from the queue */
                struct mbuf *m;
                uint32_t stop_i;

                nm_i = kring->nr_hwtail;
                stop_i = nm_prev(nm_i, lim);
                while ( nm_i != stop_i && (m = mbq_dequeue(q)) != NULL ) {
                        int len = MBUF_LEN(m);
                        struct netmap_slot *slot = &ring->slot[nm_i];

                        m_copydata(m, 0, len, NMB(na, slot));
                        ND("nm %d len %d", nm_i, len);
                        if (netmap_verbose)
                                D("%s", nm_dump_buf(NMB(na, slot),len, 128, NULL));

                        slot->len = len;
                        slot->flags = kring->nkr_slot_flags;
                        nm_i = nm_next(nm_i, lim);
                        mbq_enqueue(&fq, m);
                }
                kring->nr_hwtail = nm_i;
        }

        /*
         * Second part: skip past packets that userspace has released.
         */
        nm_i = kring->nr_hwcur;
        if (nm_i != head) { /* something was released */
                if (netmap_fwd || kring->ring->flags & NR_FORWARD)
                        ret = netmap_sw_to_nic(na);
                kring->nr_hwcur = head;
        }

        /* access copies of cur,tail in the kring */
        if (kring->rcur == kring->rtail && td) /* no bufs available */
                OS_selrecord(td, &kring->si);

        mbq_unlock(q);

        mbq_purge(&fq);
        mbq_destroy(&fq);

        return ret;
}


/* Get a netmap adapter for the port.
 *
 * If it is possible to satisfy the request, return 0
 * with *na containing the netmap adapter found.
 * Otherwise return an error code, with *na containing NULL.
 *
 * When the port is attached to a bridge, we always return
 * EBUSY.
 * Otherwise, if the port is already bound to a file descriptor,
 * then we unconditionally return the existing adapter into *na.
 * In all the other cases, we return (into *na) either native,
 * generic or NULL, according to the following table:
 *
 *                                      native_support
 * active_fds   dev.netmap.admode         YES     NO
 * -------------------------------------------------------
 *    >0              *                 NA(ifp) NA(ifp)
 *
 *     0        NETMAP_ADMODE_BEST      NATIVE  GENERIC
 *     0        NETMAP_ADMODE_NATIVE    NATIVE   NULL
 *     0        NETMAP_ADMODE_GENERIC   GENERIC GENERIC
 *
 */

int
netmap_get_hw_na(struct ifnet *ifp, struct netmap_adapter **na)
{
        /* generic support */
        int i = netmap_admode;  /* Take a snapshot. */
        struct netmap_adapter *prev_na;
#ifdef WITH_GENERIC
        struct netmap_generic_adapter *gna;
        int error = 0;
#endif

        *na = NULL; /* default */

        /* reset in case of invalid value */
        if (i < NETMAP_ADMODE_BEST || i >= NETMAP_ADMODE_LAST)
                i = netmap_admode = NETMAP_ADMODE_BEST;

        if (NETMAP_CAPABLE(ifp)) {
                prev_na = NA(ifp);
                /* If an adapter already exists, return it if
                 * there are active file descriptors or if
                 * netmap is not forced to use generic
                 * adapters.
                 */
                if (NETMAP_OWNED_BY_ANY(prev_na)
                        || i != NETMAP_ADMODE_GENERIC
                        || prev_na->na_flags & NAF_FORCE_NATIVE
#ifdef WITH_PIPES
                        /* ugly, but we cannot allow an adapter switch
                         * if some pipe is referring to this one
                         */
                        || prev_na->na_next_pipe > 0
#endif
                ) {
                        *na = prev_na;
                        return 0;
                }
        }

        /* If there isn't native support and netmap is not allowed
         * to use generic adapters, we cannot satisfy the request.
         */
        if (!NETMAP_CAPABLE(ifp) && i == NETMAP_ADMODE_NATIVE)
                return EOPNOTSUPP;

#ifdef WITH_GENERIC
        /* Otherwise, create a generic adapter and return it,
         * saving the previously used netmap adapter, if any.
         *
         * Note that here 'prev_na', if not NULL, MUST be a
         * native adapter, and CANNOT be a generic one. This is
         * true because generic adapters are created on demand, and
         * destroyed when not used anymore. Therefore, if the adapter
         * currently attached to an interface 'ifp' is generic, it
         * must be that
         * (NA(ifp)->active_fds > 0 || NETMAP_OWNED_BY_KERN(NA(ifp))).
         * Consequently, if NA(ifp) is generic, we will enter one of
         * the branches above. This ensures that we never override
         * a generic adapter with another generic adapter.
         */
        prev_na = NA(ifp);
        error = generic_netmap_attach(ifp);
        if (error)
                return error;

        *na = NA(ifp);
        gna = (struct netmap_generic_adapter*)NA(ifp);
        gna->prev = prev_na; /* save old na */
        if (prev_na != NULL) {
                ifunit_ref(ifp->if_xname);
                // XXX add a refcount ?
                netmap_adapter_get(prev_na);
        }
        ND("Created generic NA %p (prev %p)", gna, gna->prev);

        return 0;
#else /* !WITH_GENERIC */
        return EOPNOTSUPP;
#endif
}


/*
 * MUST BE CALLED UNDER NMG_LOCK()
 *
 * Get a refcounted reference to a netmap adapter attached
 * to the interface specified by nmr.
 * This is always called in the execution of an ioctl().
 *
 * Return ENXIO if the interface specified by the request does
 * not exist, ENOTSUP if netmap is not supported by the interface,
 * EBUSY if the interface is already attached to a bridge,
 * EINVAL if parameters are invalid, ENOMEM if needed resources
 * could not be allocated.
 * If successful, hold a reference to the netmap adapter.
 *
 * No reference is kept on the real interface, which may then
 * disappear at any time.
 */
int
netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na, int create)
{
        struct ifnet *ifp = NULL;
        int error = 0;
        struct netmap_adapter *ret = NULL;

        *na = NULL;     /* default return value */

        NMG_LOCK_ASSERT();

        /* we cascade through all possible types of netmap adapter.
         * All netmap_get_*_na() functions return an error and an na,
         * with the following combinations:
         *
         * error    na
         *   0     NULL         type doesn't match
         *  !0     NULL         type matches, but na creation/lookup failed
         *   0    !NULL         type matches and na created/found
         *  !0    !NULL         impossible
         */

        /* try to see if this is a monitor port */
        error = netmap_get_monitor_na(nmr, na, create);
        if (error || *na != NULL)
                return error;

        /* try to see if this is a pipe port */
        error = netmap_get_pipe_na(nmr, na, create);
        if (error || *na != NULL)
                return error;

        /* try to see if this is a bridge port */
        error = netmap_get_bdg_na(nmr, na, create);
        if (error)
                return error;

        if (*na != NULL) /* valid match in netmap_get_bdg_na() */
                goto out;

        /*
         * This must be a hardware na, lookup the name in the system.
         * Note that by hardware we actually mean "it shows up in ifconfig".
         * This may still be a tap, a veth/epair, or even a
         * persistent VALE port.
         */
        ifp = ifunit_ref(nmr->nr_name);
        if (ifp == NULL) {
                return ENXIO;
        }

        error = netmap_get_hw_na(ifp, &ret);
        if (error)
                goto out;

        *na = ret;
        netmap_adapter_get(ret);

out:
        if (error && ret != NULL)
                netmap_adapter_put(ret);

        if (ifp)
                if_rele(ifp); /* allow live unloading of drivers modules */

        return error;
}


/*
 * validate parameters on entry for *_txsync()
 * Returns ring->cur if ok, or something >= kring->nkr_num_slots
 * in case of error.
 *
 * rhead, rcur and rtail=hwtail are stored from previous round.
 * hwcur is the next packet to send to the ring.
 *
 * We want
 *    hwcur <= *rhead <= head <= cur <= tail = *rtail <= hwtail
 *
 * hwcur, rhead, rtail and hwtail are reliable
 */
static u_int
nm_txsync_prologue(struct netmap_kring *kring)
{
#define NM_ASSERT(t) if (t) { D("fail " #t); goto error; }
        struct netmap_ring *ring = kring->ring;
        u_int head = ring->head; /* read only once */
        u_int cur = ring->cur; /* read only once */
        u_int n = kring->nkr_num_slots;

        ND(5, "%s kcur %d ktail %d head %d cur %d tail %d",
                kring->name,
                kring->nr_hwcur, kring->nr_hwtail,
                ring->head, ring->cur, ring->tail);
#if 1 /* kernel sanity checks; but we can trust the kring. */
        if (kring->nr_hwcur >= n || kring->rhead >= n ||
            kring->rtail >= n ||  kring->nr_hwtail >= n)
                goto error;
#endif /* kernel sanity checks */
        /*
         * user sanity checks. We only use 'cur',
         * A, B, ... are possible positions for cur:
         *
         *  0    A  cur   B  tail  C  n-1
         *  0    D  tail  E  cur   F  n-1
         *
         * B, F, D are valid. A, C, E are wrong
         */
        if (kring->rtail >= kring->rhead) {
                /* want rhead <= head <= rtail */
                NM_ASSERT(head < kring->rhead || head > kring->rtail);
                /* and also head <= cur <= rtail */
                NM_ASSERT(cur < head || cur > kring->rtail);
        } else { /* here rtail < rhead */
                /* we need head outside rtail .. rhead */
                NM_ASSERT(head > kring->rtail && head < kring->rhead);

                /* two cases now: head <= rtail or head >= rhead  */
                if (head <= kring->rtail) {
                        /* want head <= cur <= rtail */
                        NM_ASSERT(cur < head || cur > kring->rtail);
                } else { /* head >= rhead */
                        /* cur must be outside rtail..head */
                        NM_ASSERT(cur > kring->rtail && cur < head);
                }
        }
        if (ring->tail != kring->rtail) {
                RD(5, "tail overwritten was %d need %d",
                        ring->tail, kring->rtail);
                ring->tail = kring->rtail;
        }
        kring->rhead = head;
        kring->rcur = cur;
        return head;

error:
        RD(5, "%s kring error: head %d cur %d tail %d rhead %d rcur %d rtail %d hwcur %d hwtail %d",
                kring->name,
                head, cur, ring->tail,
                kring->rhead, kring->rcur, kring->rtail,
                kring->nr_hwcur, kring->nr_hwtail);
        return n;
#undef NM_ASSERT
}


/*
 * validate parameters on entry for *_rxsync()
 * Returns ring->head if ok, kring->nkr_num_slots on error.
 *
 * For a valid configuration,
 * hwcur <= head <= cur <= tail <= hwtail
 *
 * We only consider head and cur.
 * hwcur and hwtail are reliable.
 *
 */
static u_int
nm_rxsync_prologue(struct netmap_kring *kring)
{
        struct netmap_ring *ring = kring->ring;
        uint32_t const n = kring->nkr_num_slots;
        uint32_t head, cur;

        ND(5,"%s kc %d kt %d h %d c %d t %d",
                kring->name,
                kring->nr_hwcur, kring->nr_hwtail,
                ring->head, ring->cur, ring->tail);
        /*
         * Before storing the new values, we should check they do not
         * move backwards. However:
         * - head is not an issue because the previous value is hwcur;
         * - cur could in principle go back, however it does not matter
         *   because we are processing a brand new rxsync()
         */
        cur = kring->rcur = ring->cur;  /* read only once */
        head = kring->rhead = ring->head;       /* read only once */
#if 1 /* kernel sanity checks */
        if (kring->nr_hwcur >= n || kring->nr_hwtail >= n)
                goto error;
#endif /* kernel sanity checks */
        /* user sanity checks */
        if (kring->nr_hwtail >= kring->nr_hwcur) {
                /* want hwcur <= rhead <= hwtail */
                if (head < kring->nr_hwcur || head > kring->nr_hwtail)
                        goto error;
                /* and also rhead <= rcur <= hwtail */
                if (cur < head || cur > kring->nr_hwtail)
                        goto error;
        } else {
                /* we need rhead outside hwtail..hwcur */
                if (head < kring->nr_hwcur && head > kring->nr_hwtail)
                        goto error;
                /* two cases now: head <= hwtail or head >= hwcur  */
                if (head <= kring->nr_hwtail) {
                        /* want head <= cur <= hwtail */
                        if (cur < head || cur > kring->nr_hwtail)
                                goto error;
                } else {
                        /* cur must be outside hwtail..head */
                        if (cur < head && cur > kring->nr_hwtail)
                                goto error;
                }
        }
        if (ring->tail != kring->rtail) {
                RD(5, "%s tail overwritten was %d need %d",
                        kring->name,
                        ring->tail, kring->rtail);
                ring->tail = kring->rtail;
        }
        return head;

error:
        RD(5, "kring error: hwcur %d rcur %d hwtail %d head %d cur %d tail %d",
                kring->nr_hwcur,
                kring->rcur, kring->nr_hwtail,
                kring->rhead, kring->rcur, ring->tail);
        return n;
}


/*
 * Error routine called when txsync/rxsync detects an error.
 * Can't do much more than resetting head =cur = hwcur, tail = hwtail
 * Return 1 on reinit.
 *
 * This routine is only called by the upper half of the kernel.
 * It only reads hwcur (which is changed only by the upper half, too)
 * and hwtail (which may be changed by the lower half, but only on
 * a tx ring and only to increase it, so any error will be recovered
 * on the next call). For the above, we don't strictly need to call
 * it under lock.
 */
int
netmap_ring_reinit(struct netmap_kring *kring)
{
        struct netmap_ring *ring = kring->ring;
        u_int i, lim = kring->nkr_num_slots - 1;
        int errors = 0;

        // XXX KASSERT nm_kr_tryget
        RD(10, "called for %s", kring->name);
        // XXX probably wrong to trust userspace
        kring->rhead = ring->head;
        kring->rcur  = ring->cur;
        kring->rtail = ring->tail;

        if (ring->cur > lim)
                errors++;
        if (ring->head > lim)
                errors++;
        if (ring->tail > lim)
                errors++;
        for (i = 0; i <= lim; i++) {
                u_int idx = ring->slot[i].buf_idx;
                u_int len = ring->slot[i].len;
                if (idx < 2 || idx >= kring->na->na_lut.objtotal) {
                        RD(5, "bad index at slot %d idx %d len %d ", i, idx, len);
                        ring->slot[i].buf_idx = 0;
                        ring->slot[i].len = 0;
                } else if (len > NETMAP_BUF_SIZE(kring->na)) {
                        ring->slot[i].len = 0;
                        RD(5, "bad len at slot %d idx %d len %d", i, idx, len);
                }
        }
        if (errors) {
                RD(10, "total %d errors", errors);
                RD(10, "%s reinit, cur %d -> %d tail %d -> %d",
                        kring->name,
                        ring->cur, kring->nr_hwcur,
                        ring->tail, kring->nr_hwtail);
                ring->head = kring->rhead = kring->nr_hwcur;
                ring->cur  = kring->rcur  = kring->nr_hwcur;
                ring->tail = kring->rtail = kring->nr_hwtail;
        }
        return (errors ? 1 : 0);
}

/* interpret the ringid and flags fields of an nmreq, by translating them
 * into a pair of intervals of ring indices:
 *
 * [priv->np_txqfirst, priv->np_txqlast) and
 * [priv->np_rxqfirst, priv->np_rxqlast)
 *
 */
int
netmap_interp_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags)
{
        struct netmap_adapter *na = priv->np_na;
        u_int j, i = ringid & NETMAP_RING_MASK;
        u_int reg = flags & NR_REG_MASK;
        enum txrx t;

        if (reg == NR_REG_DEFAULT) {
                /* convert from old ringid to flags */
                if (ringid & NETMAP_SW_RING) {
                        reg = NR_REG_SW;
                } else if (ringid & NETMAP_HW_RING) {
                        reg = NR_REG_ONE_NIC;
                } else {
                        reg = NR_REG_ALL_NIC;
                }
                D("deprecated API, old ringid 0x%x -> ringid %x reg %d", ringid, i, reg);
        }
        switch (reg) {
        case NR_REG_ALL_NIC:
        case NR_REG_PIPE_MASTER:
        case NR_REG_PIPE_SLAVE:
                for_rx_tx(t) {
                        priv->np_qfirst[t] = 0;
                        priv->np_qlast[t] = nma_get_nrings(na, t);
                }
                ND("%s %d %d", "ALL/PIPE",
                        priv->np_qfirst[NR_RX], priv->np_qlast[NR_RX]);
                break;
        case NR_REG_SW:
        case NR_REG_NIC_SW:
                if (!(na->na_flags & NAF_HOST_RINGS)) {
                        D("host rings not supported");
                        return EINVAL;
                }
                for_rx_tx(t) {
                        priv->np_qfirst[t] = (reg == NR_REG_SW ?
                                nma_get_nrings(na, t) : 0);
                        priv->np_qlast[t] = nma_get_nrings(na, t) + 1;
                }
                ND("%s %d %d", reg == NR_REG_SW ? "SW" : "NIC+SW",
                        priv->np_qfirst[NR_RX], priv->np_qlast[NR_RX]);
                break;
        case NR_REG_ONE_NIC:
                if (i >= na->num_tx_rings && i >= na->num_rx_rings) {
                        D("invalid ring id %d", i);
                        return EINVAL;
                }
                for_rx_tx(t) {
                        /* if not enough rings, use the first one */
                        j = i;
                        if (j >= nma_get_nrings(na, t))
                                j = 0;
                        priv->np_qfirst[t] = j;
                        priv->np_qlast[t] = j + 1;
                }
                break;
        default:
                D("invalid regif type %d", reg);
                return EINVAL;
        }
        priv->np_flags = (flags & ~NR_REG_MASK) | reg;

        if (netmap_verbose) {
                D("%s: tx [%d,%d) rx [%d,%d) id %d",
                        na->name,
                        priv->np_qfirst[NR_TX],
                        priv->np_qlast[NR_TX],
                        priv->np_qfirst[NR_RX],
                        priv->np_qlast[NR_RX],
                        i);
        }
        return 0;
}


/*
 * Set the ring ID. For devices with a single queue, a request
 * for all rings is the same as a single ring.
 */
static int
netmap_set_ringid(struct netmap_priv_d *priv, uint16_t ringid, uint32_t flags)
{
        struct netmap_adapter *na = priv->np_na;
        int error;
        enum txrx t;

        error = netmap_interp_ringid(priv, ringid, flags);
        if (error) {
                return error;
        }

        priv->np_txpoll = (ringid & NETMAP_NO_TX_POLL) ? 0 : 1;

        /* optimization: count the users registered for more than
         * one ring, which are the ones sleeping on the global queue.
         * The default netmap_notify() callback will then
         * avoid signaling the global queue if nobody is using it
         */
        for_rx_tx(t) {
                if (nm_si_user(priv, t))
                        na->si_users[t]++;
        }
        return 0;
}

static void
netmap_unset_ringid(struct netmap_priv_d *priv)
{
        struct netmap_adapter *na = priv->np_na;
        enum txrx t;

        for_rx_tx(t) {
                if (nm_si_user(priv, t))
                        na->si_users[t]--;
                priv->np_qfirst[t] = priv->np_qlast[t] = 0;
        }
        priv->np_flags = 0;
        priv->np_txpoll = 0;
}


/* check that the rings we want to bind are not exclusively owned by a previous
 * bind.  If exclusive ownership has been requested, we also mark the rings.
 */
static int
netmap_get_exclusive(struct netmap_priv_d *priv)
{
        struct netmap_adapter *na = priv->np_na;
        u_int i;
        struct netmap_kring *kring;
        int excl = (priv->np_flags & NR_EXCLUSIVE);
        enum txrx t;

        ND("%s: grabbing tx [%d, %d) rx [%d, %d)",
                        na->name,
                        priv->np_qfirst[NR_TX],
                        priv->np_qlast[NR_TX],
                        priv->np_qfirst[NR_RX],
                        priv->np_qlast[NR_RX]);

        /* first round: check that all the requested rings
         * are neither alread exclusively owned, nor we
         * want exclusive ownership when they are already in use
         */
        for_rx_tx(t) {
                for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
                        kring = &NMR(na, t)[i];
                        if ((kring->nr_kflags & NKR_EXCLUSIVE) ||
                            (kring->users && excl))
                        {
                                ND("ring %s busy", kring->name);
                                return EBUSY;
                        }
                }
        }

        /* second round: increment usage cound and possibly
         * mark as exclusive
         */

        for_rx_tx(t) {
                for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
                        kring = &NMR(na, t)[i];
                        kring->users++;
                        if (excl)
                                kring->nr_kflags |= NKR_EXCLUSIVE;
                }
        }

        return 0;

}

/* undo netmap_get_ownership() */
static void
netmap_rel_exclusive(struct netmap_priv_d *priv)
{
        struct netmap_adapter *na = priv->np_na;
        u_int i;
        struct netmap_kring *kring;
        int excl = (priv->np_flags & NR_EXCLUSIVE);
        enum txrx t;

        ND("%s: releasing tx [%d, %d) rx [%d, %d)",
                        na->name,
                        priv->np_qfirst[NR_TX],
                        priv->np_qlast[NR_TX],
                        priv->np_qfirst[NR_RX],
                        priv->np_qlast[MR_RX]);


        for_rx_tx(t) {
                for (i = priv->np_qfirst[t]; i < priv->np_qlast[t]; i++) {
                        kring = &NMR(na, t)[i];
                        if (excl)
                                kring->nr_kflags &= ~NKR_EXCLUSIVE;
                        kring->users--;
                }
        }
}

/*
 * possibly move the interface to netmap-mode.
 * If success it returns a pointer to netmap_if, otherwise NULL.
 * This must be called with NMG_LOCK held.
 *
 * The following na callbacks are called in the process:
 *
 * na->nm_config()                      [by netmap_update_config]
 * (get current number and size of rings)
 *
 *      We have a generic one for linux (netmap_linux_config).
 *      The bwrap has to override this, since it has to forward
 *      the request to the wrapped adapter (netmap_bwrap_config).
 *
 *
 * na->nm_krings_create()
 * (create and init the krings array)
 *
 *      One of the following:
 *
 *      * netmap_hw_krings_create,                      (hw ports)
 *              creates the standard layout for the krings
 *              and adds the mbq (used for the host rings).
 *
 *      * netmap_vp_krings_create                       (VALE ports)
 *              add leases and scratchpads
 *
 *      * netmap_pipe_krings_create                     (pipes)
 *              create the krings and rings of both ends and
 *              cross-link them
 *
 *      * netmap_monitor_krings_create                  (monitors)
 *              avoid allocating the mbq
 *
 *      * netmap_bwrap_krings_create                    (bwraps)
 *              create both the brap krings array,
 *              the krings array of the wrapped adapter, and
 *              (if needed) the fake array for the host adapter
 *
 * na->nm_register(, 1)
 * (put the adapter in netmap mode)
 *
 *      This may be one of the following:
 *      (XXX these should be either all *_register or all *_reg 2014-03-15)
 *
 *      * netmap_hw_register                            (hw ports)
 *              checks that the ifp is still there, then calls
 *              the hardware specific callback;
 *
 *      * netmap_vp_reg                                 (VALE ports)
 *              If the port is connected to a bridge,
 *              set the NAF_NETMAP_ON flag under the
 *              bridge write lock.
 *
 *      * netmap_pipe_reg                               (pipes)
 *              inform the other pipe end that it is no
 *              longer responsible for the lifetime of this
 *              pipe end
 *
 *      * netmap_monitor_reg                            (monitors)
 *              intercept the sync callbacks of the monitored
 *              rings
 *
 *      * netmap_bwrap_register                         (bwraps)
 *              cross-link the bwrap and hwna rings,
 *              forward the request to the hwna, override
 *              the hwna notify callback (to get the frames
 *              coming from outside go through the bridge).
 *
 *
 */
int
netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na,
        uint16_t ringid, uint32_t flags)
{
        struct netmap_if *nifp = NULL;
        int error;

        NMG_LOCK_ASSERT();
        /* ring configuration may have changed, fetch from the card */
        netmap_update_config(na);
        priv->np_na = na;     /* store the reference */
        error = netmap_set_ringid(priv, ringid, flags);
        if (error)
                goto err;
        error = netmap_mem_finalize(na->nm_mem, na);
        if (error)
                goto err;

        if (na->active_fds == 0) {
                /*
                 * If this is the first registration of the adapter,
                 * also create the netmap rings and their in-kernel view,
                 * the netmap krings.
                 */

                /*
                 * Depending on the adapter, this may also create
                 * the netmap rings themselves
                 */
                error = na->nm_krings_create(na);
                if (error)
                        goto err_drop_mem;

                /* create all missing netmap rings */
                error = netmap_mem_rings_create(na);
                if (error)
                        goto err_del_krings;
        }

        /* now the kring must exist and we can check whether some
         * previous bind has exclusive ownership on them
         */
        error = netmap_get_exclusive(priv);
        if (error)
                goto err_del_rings;

        /* in all cases, create a new netmap if */
        nifp = netmap_mem_if_new(na);
        if (nifp == NULL) {
                error = ENOMEM;
                goto err_rel_excl;
        }

        na->active_fds++;
        if (!nm_netmap_on(na)) {
                /* Netmap not active, set the card in netmap mode
                 * and make it use the shared buffers.
                 */
                /* cache the allocator info in the na */
                netmap_mem_get_lut(na->nm_mem, &na->na_lut);
                ND("%p->na_lut == %p", na, na->na_lut.lut);
                error = na->nm_register(na, 1); /* mode on */
                if (error) 
                        goto err_del_if;
        }

        /*
         * advertise that the interface is ready by setting np_nifp.
         * The barrier is needed because readers (poll, *SYNC and mmap)
         * check for priv->np_nifp != NULL without locking
         */
        mb(); /* make sure previous writes are visible to all CPUs */
        priv->np_nifp = nifp;

        return 0;

err_del_if:
        memset(&na->na_lut, 0, sizeof(na->na_lut));
        na->active_fds--;
        netmap_mem_if_delete(na, nifp);
err_rel_excl:
        netmap_rel_exclusive(priv);
err_del_rings:
        if (na->active_fds == 0)
                netmap_mem_rings_delete(na);
err_del_krings:
        if (na->active_fds == 0)
                na->nm_krings_delete(na);
err_drop_mem:
        netmap_mem_deref(na->nm_mem, na);
err:
        priv->np_na = NULL;
        return error;
}


/*
 * update kring and ring at the end of txsync.
 */
static inline void
nm_txsync_finalize(struct netmap_kring *kring)
{
        /* update ring tail to what the kernel knows */
        kring->ring->tail = kring->rtail = kring->nr_hwtail;

        /* note, head/rhead/hwcur might be behind cur/rcur
         * if no carrier
         */
        ND(5, "%s now hwcur %d hwtail %d head %d cur %d tail %d",
                kring->name, kring->nr_hwcur, kring->nr_hwtail,
                kring->rhead, kring->rcur, kring->rtail);
}


/*
 * update kring and ring at the end of rxsync
 */
static inline void
nm_rxsync_finalize(struct netmap_kring *kring)
{
        /* tell userspace that there might be new packets */
        //struct netmap_ring *ring = kring->ring;
        ND("head %d cur %d tail %d -> %d", ring->head, ring->cur, ring->tail,
                kring->nr_hwtail);
        kring->ring->tail = kring->rtail = kring->nr_hwtail;
        /* make a copy of the state for next round */
        kring->rhead = kring->ring->head;
        kring->rcur = kring->ring->cur;
}



/*
 * ioctl(2) support for the "netmap" device.
 *
 * Following a list of accepted commands:
 * - NIOCGINFO
 * - SIOCGIFADDR        just for convenience
 * - NIOCREGIF
 * - NIOCTXSYNC
 * - NIOCRXSYNC
 *
 * Return 0 on success, errno otherwise.
 */
int
netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data,
        int fflag, struct thread *td)
{
        struct netmap_priv_d *priv = NULL;
        struct nmreq *nmr = (struct nmreq *) data;
        struct netmap_adapter *na = NULL;
        int error;
        u_int i, qfirst, qlast;
        struct netmap_if *nifp;
        struct netmap_kring *krings;
        enum txrx t;

        (void)dev;      /* UNUSED */
        (void)fflag;    /* UNUSED */

        if (cmd == NIOCGINFO || cmd == NIOCREGIF) {
                /* truncate name */
                nmr->nr_name[sizeof(nmr->nr_name) - 1] = '\0';
                if (nmr->nr_version != NETMAP_API) {
                        D("API mismatch for %s got %d need %d",
                                nmr->nr_name,
                                nmr->nr_version, NETMAP_API);
                        nmr->nr_version = NETMAP_API;
                }
                if (nmr->nr_version < NETMAP_MIN_API ||
                    nmr->nr_version > NETMAP_MAX_API) {
                        return EINVAL;
                }
        }
        CURVNET_SET(TD_TO_VNET(td));

        error = devfs_get_cdevpriv((void **)&priv);
        if (error) {
                CURVNET_RESTORE();
                /* XXX ENOENT should be impossible, since the priv
                 * is now created in the open */
                return (error == ENOENT ? ENXIO : error);
        }

        switch (cmd) {
        case NIOCGINFO:         /* return capabilities etc */
                if (nmr->nr_cmd == NETMAP_BDG_LIST) {
                        error = netmap_bdg_ctl(nmr, NULL);
                        break;
                }

                NMG_LOCK();
                do {
                        /* memsize is always valid */
                        struct netmap_mem_d *nmd = &nm_mem;
                        u_int memflags;

                        if (nmr->nr_name[0] != '\0') {
                                /* get a refcount */
                                error = netmap_get_na(nmr, &na, 1 /* create */);
                                if (error)
                                        break;
                                nmd = na->nm_mem; /* get memory allocator */
                        }

                        error = netmap_mem_get_info(nmd, &nmr->nr_memsize, &memflags,
                                &nmr->nr_arg2);
                        if (error)
                                break;
                        if (na == NULL) /* only memory info */
                                break;
                        nmr->nr_offset = 0;
                        nmr->nr_rx_slots = nmr->nr_tx_slots = 0;
                        netmap_update_config(na);
                        nmr->nr_rx_rings = na->num_rx_rings;
                        nmr->nr_tx_rings = na->num_tx_rings;
                        nmr->nr_rx_slots = na->num_rx_desc;
                        nmr->nr_tx_slots = na->num_tx_desc;
                        netmap_adapter_put(na);
                } while (0);
                NMG_UNLOCK();
                break;

        case NIOCREGIF:
                /* possibly attach/detach NIC and VALE switch */
                i = nmr->nr_cmd;
                if (i == NETMAP_BDG_ATTACH || i == NETMAP_BDG_DETACH
                                || i == NETMAP_BDG_VNET_HDR
                                || i == NETMAP_BDG_NEWIF
                                || i == NETMAP_BDG_DELIF) {
                        error = netmap_bdg_ctl(nmr, NULL);
                        break;
                } else if (i != 0) {
                        D("nr_cmd must be 0 not %d", i);
                        error = EINVAL;
                        break;
                }

                /* protect access to priv from concurrent NIOCREGIF */
                NMG_LOCK();
                do {
                        u_int memflags;

                        if (priv->np_nifp != NULL) {    /* thread already registered */
                                error = EBUSY;
                                break;
                        }
                        /* find the interface and a reference */
                        error = netmap_get_na(nmr, &na, 1 /* create */); /* keep reference */
                        if (error)
                                break;
                        if (NETMAP_OWNED_BY_KERN(na)) {
                                netmap_adapter_put(na);
                                error = EBUSY;
                                break;
                        }
                        error = netmap_do_regif(priv, na, nmr->nr_ringid, nmr->nr_flags);
                        if (error) {    /* reg. failed, release priv and ref */
                                netmap_adapter_put(na);
                                break;
                        }
                        nifp = priv->np_nifp;
                        priv->np_td = td; // XXX kqueue, debugging only

                        /* return the offset of the netmap_if object */
                        nmr->nr_rx_rings = na->num_rx_rings;
                        nmr->nr_tx_rings = na->num_tx_rings;
                        nmr->nr_rx_slots = na->num_rx_desc;
                        nmr->nr_tx_slots = na->num_tx_desc;
                        error = netmap_mem_get_info(na->nm_mem, &nmr->nr_memsize, &memflags,
                                &nmr->nr_arg2);
                        if (error) {
                                netmap_do_unregif(priv);
                                netmap_adapter_put(na);
                                break;
                        }
                        if (memflags & NETMAP_MEM_PRIVATE) {
                                *(uint32_t *)(uintptr_t)&nifp->ni_flags |= NI_PRIV_MEM;
                        }
                        for_rx_tx(t) {
                                priv->np_si[t] = nm_si_user(priv, t) ?
                                        &na->si[t] : &NMR(na, t)[priv->np_qfirst[t]].si;
                        }

                        if (nmr->nr_arg3) {
                                D("requested %d extra buffers", nmr->nr_arg3);
                                nmr->nr_arg3 = netmap_extra_alloc(na,
                                        &nifp->ni_bufs_head, nmr->nr_arg3);
                                D("got %d extra buffers", nmr->nr_arg3);
                        }
                        nmr->nr_offset = netmap_mem_if_offset(na->nm_mem, nifp);
                } while (0);
                NMG_UNLOCK();
                break;

        case NIOCTXSYNC:
        case NIOCRXSYNC:
                nifp = priv->np_nifp;

                if (nifp == NULL) {
                        error = ENXIO;
                        break;
                }
                mb(); /* make sure following reads are not from cache */

                na = priv->np_na;      /* we have a reference */

                if (na == NULL) {
                        D("Internal error: nifp != NULL && na == NULL");
                        error = ENXIO;
                        break;
                }

                if (!nm_netmap_on(na)) {
                        error = ENXIO;
                        break;
                }

                t = (cmd == NIOCTXSYNC ? NR_TX : NR_RX);
                krings = NMR(na, t);
                qfirst = priv->np_qfirst[t];
                qlast = priv->np_qlast[t];

                for (i = qfirst; i < qlast; i++) {
                        struct netmap_kring *kring = krings + i;
                        if (nm_kr_tryget(kring)) {
                                error = EBUSY;
                                goto out;
                        }
                        if (cmd == NIOCTXSYNC) {
                                if (netmap_verbose & NM_VERB_TXSYNC)
                                        D("pre txsync ring %d cur %d hwcur %d",
                                            i, kring->ring->cur,
                                            kring->nr_hwcur);
                                if (nm_txsync_prologue(kring) >= kring->nkr_num_slots) {
                                        netmap_ring_reinit(kring);
                                } else if (kring->nm_sync(kring, NAF_FORCE_RECLAIM) == 0) {
                                        nm_txsync_finalize(kring);
                                }
                                if (netmap_verbose & NM_VERB_TXSYNC)
                                        D("post txsync ring %d cur %d hwcur %d",
                                            i, kring->ring->cur,
                                            kring->nr_hwcur);
                        } else {
                                if (nm_rxsync_prologue(kring) >= kring->nkr_num_slots) {
                                        netmap_ring_reinit(kring);
                                } else if (kring->nm_sync(kring, NAF_FORCE_READ) == 0) {
                                        nm_rxsync_finalize(kring);
                                }
                                microtime(&na->rx_rings[i].ring->ts);
                        }
                        nm_kr_put(kring);
                }

                break;

#ifdef WITH_VALE
        case NIOCCONFIG:
                error = netmap_bdg_config(nmr);
                break;
#endif
#ifdef __FreeBSD__
        case FIONBIO:
        case FIOASYNC:
                ND("FIONBIO/FIOASYNC are no-ops");
                break;

        case BIOCIMMEDIATE:
        case BIOCGHDRCMPLT:
        case BIOCSHDRCMPLT:
        case BIOCSSEESENT:
                D("ignore BIOCIMMEDIATE/BIOCSHDRCMPLT/BIOCSHDRCMPLT/BIOCSSEESENT");
                break;

        default:        /* allow device-specific ioctls */
            {
                struct ifnet *ifp = ifunit_ref(nmr->nr_name);
                if (ifp == NULL) {
                        error = ENXIO;
                } else {
                        struct socket so;

                        bzero(&so, sizeof(so));
                        so.so_vnet = ifp->if_vnet;
                        // so->so_proto not null.
                        error = ifioctl(&so, cmd, data, td);
                        if_rele(ifp);
                }
                break;
            }

#else /* linux */
        default:
                error = EOPNOTSUPP;
#endif /* linux */
        }
out:

        CURVNET_RESTORE();
        return (error);
}


/*
 * select(2) and poll(2) handlers for the "netmap" device.
 *
 * Can be called for one or more queues.
 * Return true the event mask corresponding to ready events.
 * If there are no ready events, do a selrecord on either individual
 * selinfo or on the global one.
 * Device-dependent parts (locking and sync of tx/rx rings)
 * are done through callbacks.
 *
 * On linux, arguments are really pwait, the poll table, and 'td' is struct file *
 * The first one is remapped to pwait as selrecord() uses the name as an
 * hidden argument.
 */
int
netmap_poll(struct cdev *dev, int events, struct thread *td)
{
        struct netmap_priv_d *priv = NULL;
        struct netmap_adapter *na;
        struct netmap_kring *kring;
        u_int i, check_all_tx, check_all_rx, want[NR_TXRX], revents = 0;
#define want_tx want[NR_TX]
#define want_rx want[NR_RX]
        struct mbq q;           /* packets from hw queues to host stack */
        void *pwait = dev;      /* linux compatibility */
        int is_kevent = 0;
        enum txrx t;

        /*
         * In order to avoid nested locks, we need to "double check"
         * txsync and rxsync if we decide to do a selrecord().
         * retry_tx (and retry_rx, later) prevent looping forever.
         */
        int retry_tx = 1, retry_rx = 1;

        (void)pwait;
        mbq_init(&q);

        /*
         * XXX kevent has curthread->tp_fop == NULL,
         * so devfs_get_cdevpriv() fails. We circumvent this by passing
         * priv as the first argument, which is also useful to avoid
         * the selrecord() which are not necessary in that case.
         */
        if (devfs_get_cdevpriv((void **)&priv) != 0) {
                is_kevent = 1;
                if (netmap_verbose)
                        D("called from kevent");
                priv = (struct netmap_priv_d *)dev;
        }
        if (priv == NULL)
                return POLLERR;

        if (priv->np_nifp == NULL) {
                D("No if registered");
                return POLLERR;
        }
        mb(); /* make sure following reads are not from cache */

        na = priv->np_na;

        if (!nm_netmap_on(na))
                return POLLERR;

        if (netmap_verbose & 0x8000)
                D("device %s events 0x%x", na->name, events);
        want_tx = events & (POLLOUT | POLLWRNORM);
        want_rx = events & (POLLIN | POLLRDNORM);


        /*
         * check_all_{tx|rx} are set if the card has more than one queue AND
         * the file descriptor is bound to all of them. If so, we sleep on
         * the "global" selinfo, otherwise we sleep on individual selinfo
         * (FreeBSD only allows two selinfo's per file descriptor).
         * The interrupt routine in the driver wake one or the other
         * (or both) depending on which clients are active.
         *
         * rxsync() is only called if we run out of buffers on a POLLIN.
         * txsync() is called if we run out of buffers on POLLOUT, or
         * there are pending packets to send. The latter can be disabled
         * passing NETMAP_NO_TX_POLL in the NIOCREG call.
         */
        check_all_tx = nm_si_user(priv, NR_TX);
        check_all_rx = nm_si_user(priv, NR_RX);

        /*
         * We start with a lock free round which is cheap if we have
         * slots available. If this fails, then lock and call the sync
         * routines.
         */
        for_rx_tx(t) {
                for (i = priv->np_qfirst[t]; want[t] && i < priv->np_qlast[t]; i++) {
                        kring = &NMR(na, t)[i];
                        /* XXX compare ring->cur and kring->tail */
                        if (!nm_ring_empty(kring->ring)) {
                                revents |= want[t];
                                want[t] = 0;    /* also breaks the loop */
                        }
                }
        }

        /*
         * If we want to push packets out (priv->np_txpoll) or
         * want_tx is still set, we must issue txsync calls
         * (on all rings, to avoid that the tx rings stall).
         * XXX should also check cur != hwcur on the tx rings.
         * Fortunately, normal tx mode has np_txpoll set.
         */
        if (priv->np_txpoll || want_tx) {
                /*
                 * The first round checks if anyone is ready, if not
                 * do a selrecord and another round to handle races.
                 * want_tx goes to 0 if any space is found, and is
                 * used to skip rings with no pending transmissions.
                 */
flush_tx:
                for (i = priv->np_qfirst[NR_TX]; i < priv->np_qlast[NR_RX]; i++) {
                        int found = 0;

                        kring = &na->tx_rings[i];
                        if (!want_tx && kring->ring->cur == kring->nr_hwcur)
                                continue;
                        /* only one thread does txsync */
                        if (nm_kr_tryget(kring)) {
                                /* either busy or stopped
                                 * XXX if the ring is stopped, sleeping would
                                 * be better. In current code, however, we only
                                 * stop the rings for brief intervals (2014-03-14)
                                 */
                                if (netmap_verbose)
                                        RD(2, "%p lost race on txring %d, ok",
                                            priv, i);
                                continue;
                        }
                        if (nm_txsync_prologue(kring) >= kring->nkr_num_slots) {
                                netmap_ring_reinit(kring);
                                revents |= POLLERR;
                        } else {
                                if (kring->nm_sync(kring, 0))
                                        revents |= POLLERR;
                                else
                                        nm_txsync_finalize(kring);
                        }

                        /*
                         * If we found new slots, notify potential
                         * listeners on the same ring.
                         * Since we just did a txsync, look at the copies
                         * of cur,tail in the kring.
                         */
                        found = kring->rcur != kring->rtail;
                        nm_kr_put(kring);
                        if (found) { /* notify other listeners */
                                revents |= want_tx;
                                want_tx = 0;
                                kring->nm_notify(kring, 0);
                        }
                }
                if (want_tx && retry_tx && !is_kevent) {
                        OS_selrecord(td, check_all_tx ?
                            &na->si[NR_TX] : &na->tx_rings[priv->np_qfirst[NR_TX]].si);
                        retry_tx = 0;
                        goto flush_tx;
                }
        }

        /*
         * If want_rx is still set scan receive rings.
         * Do it on all rings because otherwise we starve.
         */
        if (want_rx) {
                int send_down = 0; /* transparent mode */
                /* two rounds here for race avoidance */
do_retry_rx:
                for (i = priv->np_qfirst[NR_RX]; i < priv->np_qlast[NR_RX]; i++) {
                        int found = 0;

                        kring = &na->rx_rings[i];

                        if (nm_kr_tryget(kring)) {
                                if (netmap_verbose)
                                        RD(2, "%p lost race on rxring %d, ok",
                                            priv, i);
                                continue;
                        }

                        if (nm_rxsync_prologue(kring) >= kring->nkr_num_slots) {
                                netmap_ring_reinit(kring);
                                revents |= POLLERR;
                        }
                        /* now we can use kring->rcur, rtail */

                        /*
                         * transparent mode support: collect packets
                         * from the rxring(s).
                         * XXX NR_FORWARD should only be read on
                         * physical or NIC ports
                         */
                        if (netmap_fwd ||kring->ring->flags & NR_FORWARD) {
                                ND(10, "forwarding some buffers up %d to %d",
                                    kring->nr_hwcur, kring->ring->cur);
                                netmap_grab_packets(kring, &q, netmap_fwd);
                        }

                        if (kring->nm_sync(kring, 0))
                                revents |= POLLERR;
                        else
                                nm_rxsync_finalize(kring);
                        if (netmap_no_timestamp == 0 ||
                                        kring->ring->flags & NR_TIMESTAMP) {
                                microtime(&kring->ring->ts);
                        }
                        found = kring->rcur != kring->rtail;
                        nm_kr_put(kring);
                        if (found) {
                                revents |= want_rx;
                                retry_rx = 0;
                                kring->nm_notify(kring, 0);
                        }
                }

                /* transparent mode XXX only during first pass ? */
                if (na->na_flags & NAF_HOST_RINGS) {
                        kring = &na->rx_rings[na->num_rx_rings];
                        if (check_all_rx
                            && (netmap_fwd || kring->ring->flags & NR_FORWARD)) {
                                /* XXX fix to use kring fields */
                                if (nm_ring_empty(kring->ring))
                                        send_down = netmap_rxsync_from_host(na, td, dev);
                                if (!nm_ring_empty(kring->ring))
                                        revents |= want_rx;
                        }
                }

                if (retry_rx && !is_kevent)
                        OS_selrecord(td, check_all_rx ?
                            &na->si[NR_RX] : &na->rx_rings[priv->np_qfirst[NR_RX]].si);
                if (send_down > 0 || retry_rx) {
                        retry_rx = 0;
                        if (send_down)
                                goto flush_tx; /* and retry_rx */
                        else
                                goto do_retry_rx;
                }
        }

        /*
         * Transparent mode: marked bufs on rx rings between
         * kring->nr_hwcur and ring->head
         * are passed to the other endpoint.
         *
         * In this mode we also scan the sw rxring, which in
         * turn passes packets up.
         *
         * XXX Transparent mode at the moment requires to bind all
         * rings to a single file descriptor.
         */

        if (q.head && na->ifp != NULL)
                netmap_send_up(na->ifp, &q);

        return (revents);
#undef want_tx
#undef want_rx
}


/*-------------------- driver support routines -------------------*/

static int netmap_hw_krings_create(struct netmap_adapter *);

/* default notify callback */
static int
netmap_notify(struct netmap_kring *kring, int flags)
{
        struct netmap_adapter *na = kring->na;
        enum txrx t = kring->tx;

        OS_selwakeup(&kring->si, PI_NET);
        /* optimization: avoid a wake up on the global
         * queue if nobody has registered for more
         * than one ring
         */
        if (na->si_users[t] > 0)
                OS_selwakeup(&na->si[t], PI_NET);

        return 0;
}


/* called by all routines that create netmap_adapters.
 * Attach na to the ifp (if any) and provide defaults
 * for optional callbacks. Defaults assume that we
 * are creating an hardware netmap_adapter.
 */
int
netmap_attach_common(struct netmap_adapter *na)
{
        struct ifnet *ifp = na->ifp;

        if (na->num_tx_rings == 0 || na->num_rx_rings == 0) {
                D("%s: invalid rings tx %d rx %d",
                        na->name, na->num_tx_rings, na->num_rx_rings);
                return EINVAL;
        }
        /* ifp is NULL for virtual adapters (bwrap, non-persistent VALE ports,
         * pipes, monitors). For bwrap we actually have a non-null ifp for
         * use by the external modules, but that is set after this
         * function has been called.
         * XXX this is ugly, maybe split this function in two (2014-03-14)
         */
        if (ifp != NULL) {
                WNA(ifp) = na;

        /* the following is only needed for na that use the host port.
         * XXX do we have something similar for linux ?
         */
#ifdef __FreeBSD__
                na->if_input = ifp->if_input; /* for netmap_send_up */
#endif /* __FreeBSD__ */

                NETMAP_SET_CAPABLE(ifp);
        }
        if (na->nm_krings_create == NULL) {
                /* we assume that we have been called by a driver,
                 * since other port types all provide their own
                 * nm_krings_create
                 */
                na->nm_krings_create = netmap_hw_krings_create;
                na->nm_krings_delete = netmap_hw_krings_delete;
        }
        if (na->nm_notify == NULL)
                na->nm_notify = netmap_notify;
        na->active_fds = 0;

        if (na->nm_mem == NULL)
                /* use the global allocator */
                na->nm_mem = &nm_mem;
        netmap_mem_get(na->nm_mem);
#ifdef WITH_VALE
        if (na->nm_bdg_attach == NULL)
                /* no special nm_bdg_attach callback. On VALE
                 * attach, we need to interpose a bwrap
                 */
                na->nm_bdg_attach = netmap_bwrap_attach;
#endif
        return 0;
}


/* standard cleanup, called by all destructors */
void
netmap_detach_common(struct netmap_adapter *na)
{
        if (na->ifp != NULL)
                WNA(na->ifp) = NULL; /* XXX do we need this? */

        if (na->tx_rings) { /* XXX should not happen */
                D("freeing leftover tx_rings");
                na->nm_krings_delete(na);
        }
        netmap_pipe_dealloc(na);
        if (na->nm_mem)
                netmap_mem_put(na->nm_mem);
        bzero(na, sizeof(*na));
        free(na, M_DEVBUF);
}

/* Wrapper for the register callback provided hardware drivers.
 * na->ifp == NULL means the driver module has been
 * unloaded, so we cannot call into it.
 * Note that module unloading, in our patched linux drivers,
 * happens under NMG_LOCK and after having stopped all the
 * nic rings (see netmap_detach). This provides sufficient
 * protection for the other driver-provied callbacks
 * (i.e., nm_config and nm_*xsync), that therefore don't need
 * to wrapped.
 */
static int
netmap_hw_register(struct netmap_adapter *na, int onoff)
{
        struct netmap_hw_adapter *hwna =
                (struct netmap_hw_adapter*)na;

        if (na->ifp == NULL)
                return onoff ? ENXIO : 0;

        return hwna->nm_hw_register(na, onoff);
}


/*
 * Initialize a ``netmap_adapter`` object created by driver on attach.
 * We allocate a block of memory with room for a struct netmap_adapter
 * plus two sets of N+2 struct netmap_kring (where N is the number
 * of hardware rings):
 * krings       0..N-1  are for the hardware queues.
 * kring        N       is for the host stack queue
 * kring        N+1     is only used for the selinfo for all queues. // XXX still true ?
 * Return 0 on success, ENOMEM otherwise.
 */
int
netmap_attach(struct netmap_adapter *arg)
{
        struct netmap_hw_adapter *hwna = NULL;
        // XXX when is arg == NULL ?
        struct ifnet *ifp = arg ? arg->ifp : NULL;

        if (arg == NULL || ifp == NULL)
                goto fail;
        hwna = malloc(sizeof(*hwna), M_DEVBUF, M_NOWAIT | M_ZERO);
        if (hwna == NULL)
                goto fail;
        hwna->up = *arg;
        hwna->up.na_flags |= NAF_HOST_RINGS | NAF_NATIVE;
        strncpy(hwna->up.name, ifp->if_xname, sizeof(hwna->up.name));
        hwna->nm_hw_register = hwna->up.nm_register;
        hwna->up.nm_register = netmap_hw_register;
        if (netmap_attach_common(&hwna->up)) {
                free(hwna, M_DEVBUF);
                goto fail;
        }
        netmap_adapter_get(&hwna->up);

#ifdef linux
        if (ifp->netdev_ops) {
                /* prepare a clone of the netdev ops */
#ifndef NETMAP_LINUX_HAVE_NETDEV_OPS
                hwna->nm_ndo.ndo_start_xmit = ifp->netdev_ops;
#else
                hwna->nm_ndo = *ifp->netdev_ops;
#endif
        }
        hwna->nm_ndo.ndo_start_xmit = linux_netmap_start_xmit;
        if (ifp->ethtool_ops) {
                hwna->nm_eto = *ifp->ethtool_ops;
        }
        hwna->nm_eto.set_ringparam = linux_netmap_set_ringparam;
#ifdef NETMAP_LINUX_HAVE_SET_CHANNELS
        hwna->nm_eto.set_channels = linux_netmap_set_channels;
#endif
        if (arg->nm_config == NULL) {
                hwna->up.nm_config = netmap_linux_config;
        }
#endif /* linux */

        if_printf(ifp, "netmap queues/slots: TX %d/%d, RX %d/%d\n",
            hwna->up.num_tx_rings, hwna->up.num_tx_desc,
            hwna->up.num_rx_rings, hwna->up.num_rx_desc);
        return 0;

fail:
        D("fail, arg %p ifp %p na %p", arg, ifp, hwna);
        if (ifp)
                netmap_detach(ifp);
        return (hwna ? EINVAL : ENOMEM);
}


void
NM_DBG(netmap_adapter_get)(struct netmap_adapter *na)
{
        if (!na) {
                return;
        }

        refcount_acquire(&na->na_refcount);
}


/* returns 1 iff the netmap_adapter is destroyed */
int
NM_DBG(netmap_adapter_put)(struct netmap_adapter *na)
{
        if (!na)
                return 1;

        if (!refcount_release(&na->na_refcount))
                return 0;

        if (na->nm_dtor)
                na->nm_dtor(na);

        netmap_detach_common(na);

        return 1;
}

/* nm_krings_create callback for all hardware native adapters */
int
netmap_hw_krings_create(struct netmap_adapter *na)
{
        int ret = netmap_krings_create(na, 0);
        if (ret == 0) {
                /* initialize the mbq for the sw rx ring */
                mbq_safe_init(&na->rx_rings[na->num_rx_rings].rx_queue);
                ND("initialized sw rx queue %d", na->num_rx_rings);
        }
        return ret;
}



/*
 * Called on module unload by the netmap-enabled drivers
 */
void
netmap_detach(struct ifnet *ifp)
{
        struct netmap_adapter *na = NA(ifp);
        int skip;

        if (!na)
                return;

        skip = 0;
        NMG_LOCK();
        netmap_disable_all_rings(ifp);
        na->ifp = NULL;
        na->na_flags &= ~NAF_NETMAP_ON;
        /*
         * if the netmap adapter is not native, somebody
         * changed it, so we can not release it here.
         * The NULL na->ifp will notify the new owner that
         * the driver is gone.
         */
        if (na->na_flags & NAF_NATIVE) {
                skip = netmap_adapter_put(na);
        }
        /* give them a chance to notice */
        if (skip == 0)
                netmap_enable_all_rings(ifp);
        NMG_UNLOCK();
}


/*
 * Intercept packets from the network stack and pass them
 * to netmap as incoming packets on the 'software' ring.
 *
 * We only store packets in a bounded mbq and then copy them
 * in the relevant rxsync routine.
 *
 * We rely on the OS to make sure that the ifp and na do not go
 * away (typically the caller checks for IFF_DRV_RUNNING or the like).
 * In nm_register() or whenever there is a reinitialization,
 * we make sure to make the mode change visible here.
 */
int
netmap_transmit(struct ifnet *ifp, struct mbuf *m)
{
        struct netmap_adapter *na = NA(ifp);
        struct netmap_kring *kring;
        u_int len = MBUF_LEN(m);
        u_int error = ENOBUFS;
        struct mbq *q;
        int space;

        kring = &na->rx_rings[na->num_rx_rings];
        // XXX [Linux] we do not need this lock
        // if we follow the down/configure/up protocol -gl
        // mtx_lock(&na->core_lock);

        if (!nm_netmap_on(na)) {
                D("%s not in netmap mode anymore", na->name);
                error = ENXIO;
                goto done;
        }

        q = &kring->rx_queue;

        // XXX reconsider long packets if we handle fragments
        if (len > NETMAP_BUF_SIZE(na)) { /* too long for us */
                D("%s from_host, drop packet size %d > %d", na->name,
                        len, NETMAP_BUF_SIZE(na));
                goto done;
        }

        /* protect against rxsync_from_host(), netmap_sw_to_nic()
         * and maybe other instances of netmap_transmit (the latter
         * not possible on Linux).
         * Also avoid overflowing the queue.
         */
        mbq_lock(q);

        space = kring->nr_hwtail - kring->nr_hwcur;
        if (space < 0)
                space += kring->nkr_num_slots;
        if (space + mbq_len(q) >= kring->nkr_num_slots - 1) { // XXX
                RD(10, "%s full hwcur %d hwtail %d qlen %d len %d m %p",
                        na->name, kring->nr_hwcur, kring->nr_hwtail, mbq_len(q),
                        len, m);
        } else {
                mbq_enqueue(q, m);
                ND(10, "%s %d bufs in queue len %d m %p",
                        na->name, mbq_len(q), len, m);
                /* notify outside the lock */
                m = NULL;
                error = 0;
        }
        mbq_unlock(q);

done:
        if (m)
                m_freem(m);
        /* unconditionally wake up listeners */
        kring->nm_notify(kring, 0);
        /* this is normally netmap_notify(), but for nics
         * connected to a bridge it is netmap_bwrap_intr_notify(),
         * that possibly forwards the frames through the switch
         */

        return (error);
}


/*
 * netmap_reset() is called by the driver routines when reinitializing
 * a ring. The driver is in charge of locking to protect the kring.
 * If native netmap mode is not set just return NULL.
 */
struct netmap_slot *
netmap_reset(struct netmap_adapter *na, enum txrx tx, u_int n,
        u_int new_cur)
{
        struct netmap_kring *kring;
        int new_hwofs, lim;

        if (!nm_native_on(na)) {
                ND("interface not in native netmap mode");
                return NULL;    /* nothing to reinitialize */
        }

        /* XXX note- in the new scheme, we are not guaranteed to be
         * under lock (e.g. when called on a device reset).
         * In this case, we should set a flag and do not trust too
         * much the values. In practice: TODO
         * - set a RESET flag somewhere in the kring
         * - do the processing in a conservative way
         * - let the *sync() fixup at the end.
         */
        if (tx == NR_TX) {
                if (n >= na->num_tx_rings)
                        return NULL;
                kring = na->tx_rings + n;
                // XXX check whether we should use hwcur or rcur
                new_hwofs = kring->nr_hwcur - new_cur;
        } else {
                if (n >= na->num_rx_rings)
                        return NULL;
                kring = na->rx_rings + n;
                new_hwofs = kring->nr_hwtail - new_cur;
        }
        lim = kring->nkr_num_slots - 1;
        if (new_hwofs > lim)
                new_hwofs -= lim + 1;

        /* Always set the new offset value and realign the ring. */
        if (netmap_verbose)
            D("%s %s%d hwofs %d -> %d, hwtail %d -> %d",
                na->name,
                tx == NR_TX ? "TX" : "RX", n,
                kring->nkr_hwofs, new_hwofs,
                kring->nr_hwtail,
                tx == NR_TX ? lim : kring->nr_hwtail);
        kring->nkr_hwofs = new_hwofs;
        if (tx == NR_TX) {
                kring->nr_hwtail = kring->nr_hwcur + lim;
                if (kring->nr_hwtail > lim)
                        kring->nr_hwtail -= lim + 1;
        }

#if 0 // def linux
        /* XXX check that the mappings are correct */
        /* need ring_nr, adapter->pdev, direction */
        buffer_info->dma = dma_map_single(&pdev->dev, addr, adapter->rx_buffer_len, DMA_FROM_DEVICE);
        if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) {
                D("error mapping rx netmap buffer %d", i);
                // XXX fix error handling
        }

#endif /* linux */
        /*
         * Wakeup on the individual and global selwait
         * We do the wakeup here, but the ring is not yet reconfigured.
         * However, we are under lock so there are no races.
         */
        kring->nm_notify(kring, 0);
        return kring->ring->slot;
}


/*
 * Dispatch rx/tx interrupts to the netmap rings.
 *
 * "work_done" is non-null on the RX path, NULL for the TX path.
 * We rely on the OS to make sure that there is only one active
 * instance per queue, and that there is appropriate locking.
 *
 * The 'notify' routine depends on what the ring is attached to.
 * - for a netmap file descriptor, do a selwakeup on the individual
 *   waitqueue, plus one on the global one if needed
 *   (see netmap_notify)
 * - for a nic connected to a switch, call the proper forwarding routine
 *   (see netmap_bwrap_intr_notify)
 */
void
netmap_common_irq(struct ifnet *ifp, u_int q, u_int *work_done)
{
        struct netmap_adapter *na = NA(ifp);
        struct netmap_kring *kring;
        enum txrx t = (work_done ? NR_RX : NR_TX);

        q &= NETMAP_RING_MASK;

        if (netmap_verbose) {
                RD(5, "received %s queue %d", work_done ? "RX" : "TX" , q);
        }

        if (q >= nma_get_nrings(na, t))
                return; // not a physical queue

        kring = NMR(na, t) + q;

        if (t == NR_RX) {
                kring->nr_kflags |= NKR_PENDINTR;       // XXX atomic ?
                *work_done = 1; /* do not fire napi again */
        }
        kring->nm_notify(kring, 0);
}


/*
 * Default functions to handle rx/tx interrupts from a physical device.
 * "work_done" is non-null on the RX path, NULL for the TX path.
 *
 * If the card is not in netmap mode, simply return 0,
 * so that the caller proceeds with regular processing.
 * Otherwise call netmap_common_irq() and return 1.
 *
 * If the card is connected to a netmap file descriptor,
 * do a selwakeup on the individual queue, plus one on the global one
 * if needed (multiqueue card _and_ there are multiqueue listeners),
 * and return 1.
 *
 * Finally, if called on rx from an interface connected to a switch,
 * calls the proper forwarding routine, and return 1.
 */
int
netmap_rx_irq(struct ifnet *ifp, u_int q, u_int *work_done)
{
        struct netmap_adapter *na = NA(ifp);

        /*
         * XXX emulated netmap mode sets NAF_SKIP_INTR so
         * we still use the regular driver even though the previous
         * check fails. It is unclear whether we should use
         * nm_native_on() here.
         */
        if (!nm_netmap_on(na))
                return 0;

        if (na->na_flags & NAF_SKIP_INTR) {
                ND("use regular interrupt");
                return 0;
        }

        netmap_common_irq(ifp, q, work_done);
        return 1;
}


/*
 * Module loader and unloader
 *
 * netmap_init() creates the /dev/netmap device and initializes
 * all global variables. Returns 0 on success, errno on failure
 * (but there is no chance)
 *
 * netmap_fini() destroys everything.
 */

static struct cdev *netmap_dev; /* /dev/netmap character device. */
extern struct cdevsw netmap_cdevsw;


void
netmap_fini(void)
{
        netmap_uninit_bridges();
        if (netmap_dev)
                destroy_dev(netmap_dev);
        netmap_mem_fini();
        NMG_LOCK_DESTROY();
        printf("netmap: unloaded module.\n");
}


int
netmap_init(void)
{
        int error;

        NMG_LOCK_INIT();

        error = netmap_mem_init();
        if (error != 0)
                goto fail;
        /*
         * MAKEDEV_ETERNAL_KLD avoids an expensive check on syscalls
         * when the module is compiled in.
         * XXX could use make_dev_credv() to get error number
         */
        netmap_dev = make_dev_credf(MAKEDEV_ETERNAL_KLD,
                &netmap_cdevsw, 0, NULL, UID_ROOT, GID_WHEEL, 0600,
                              "netmap");
        if (!netmap_dev)
                goto fail;

        error = netmap_init_bridges();
        if (error)
                goto fail;

#ifdef __FreeBSD__
        nm_vi_init_index();
#endif

        printf("netmap: loaded module\n");
        return (0);
fail:
        netmap_fini();
        return (EINVAL); /* may be incorrect */
}