sys/contrib/rdma/krping/krping.h

   1 /*
   2  * $FreeBSD$
   3  */
   4 #include <rdma/ib_verbs.h>
   5 #include <netinet/in.h>
   6
   7 /*
   8  * Krping header stuffs...
   9  */
  10
  11 struct krping_stats {
  12         unsigned send_bytes;
  13         unsigned send_msgs;
  14         unsigned recv_bytes;
  15         unsigned recv_msgs;
  16         unsigned write_bytes;
  17         unsigned write_msgs;
  18         unsigned read_bytes;
  19         unsigned read_msgs;
  20 };
  21
  22
  23 /*
  24  * These states are used to signal events between the completion handler
  25  * and the main client or server thread.
  26  *
  27  * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV,
  28  * and RDMA_WRITE_COMPLETE for each ping.
  29  */
  30 enum test_state {
  31         IDLE = 1,
  32         CONNECT_REQUEST,
  33         ADDR_RESOLVED,
  34         ROUTE_RESOLVED,
  35         CONNECTED,
  36         RDMA_READ_ADV,
  37         RDMA_READ_COMPLETE,
  38         RDMA_WRITE_ADV,
  39         RDMA_WRITE_COMPLETE,
  40         ERROR
  41 };
  42
  43 struct krping_rdma_info {
  44         uint64_t buf;
  45         uint32_t rkey;
  46         uint32_t size;
  47 };
  48
  49 /*
  50  * Control block struct.
  51  */
  52 struct krping_cb {
  53         int server;                     /* 0 iff client */
  54         struct ib_cq *cq;
  55         struct ib_pd *pd;
  56         struct ib_qp *qp;
  57         struct ib_mr *dma_mr;
  58         int use_dmamr;
  59
  60         struct ib_recv_wr rq_wr;        /* recv work request record */
  61         struct ib_sge recv_sgl;         /* recv single SGE */
  62         struct krping_rdma_info recv_buf;/* malloc'd buffer */
  63         struct ib_mr *recv_mr;
  64
  65         struct ib_send_wr sq_wr;        /* send work requrest record */
  66         struct ib_sge send_sgl;
  67         struct krping_rdma_info send_buf;/* single send buf */
  68         struct ib_mr *send_mr;
  69
  70         struct ib_send_wr rdma_sq_wr;   /* rdma work request record */
  71         struct ib_sge rdma_sgl;         /* rdma single SGE */
  72         char *rdma_buf;                 /* used as rdma sink */
  73         u64  rdma_addr;
  74         struct ib_mr *rdma_mr;
  75
  76         uint32_t remote_rkey;           /* remote guys RKEY */
  77         uint64_t remote_addr;           /* remote guys TO */
  78         uint32_t remote_len;            /* remote guys LEN */
  79
  80         char *start_buf;                /* rdma read src */
  81         u64  start_addr;
  82         struct ib_mr *start_mr;
  83
  84         enum test_state state;          /* used for cond/signalling */
  85         struct mtx lock;
  86         struct krping_stats stats;
  87
  88         uint16_t port;                  /* dst port in NBO */
  89         struct in_addr addr;            /* dst addr in NBO */
  90         char *addr_str;                 /* dst addr string */
  91         int verbose;                    /* verbose logging */
  92         int count;                      /* ping count */
  93         int size;                       /* ping data size */
  94         int validate;                   /* validate ping data */
  95         uint64_t memlimit;              /* limit of the physical memory that
  96                                            can be registered with dma_mr mode */
  97
  98         /* CM stuff */
  99         struct rdma_cm_id *cm_id;       /* connection on client side,*/
 100                                         /* listener on service side. */
 101         struct rdma_cm_id *child_cm_id; /* connection on server side */
 102         TAILQ_ENTRY(krping_cb) list;
 103
 104         int rlat;                       /* run read latency test */
 105         int wlat;                       /* run write latency test */
 106         int bw;                         /* run write bw test */
 107         int duplex;                     /* run write bw full duplex test */
 108         int poll;                       /* poll vs block in rlat */
 109         int txdepth;
 110 };
 111
 112 static __inline uint64_t
 113 get_cycles(void)
 114 {
 115         u_int32_t low, high;
 116         __asm __volatile("rdtsc" : "=a" (low), "=d" (high));
 117         return (low | ((u_int64_t)high << 32));
 118 }
 119
 120 #define htonll(x) htobe64((x))
 121 #define ntohll(x) be64toh((x))
 122
 123 typedef uint64_t cycles_t;
 124
 125 extern struct mtx krping_mutex;
 126 TAILQ_HEAD(krping_cb_list, krping_cb);
 127 extern struct krping_cb_list krping_cbs;
 128
 129 int krping_doit(char *cmd);
 130 void krping_init(void);