sys/contrib/rdma/krping/krping.h

   1 /*
   2  * $FreeBSD$
   3  */
   4 #include <rdma/ib_verbs.h>
   5 #include <netinet/in.h>
   6
   7 /*
   8  * Krping header stuffs...
   9  */
  10
  11 struct krping_stats {
  12         unsigned send_bytes;
  13         unsigned send_msgs;
  14         unsigned recv_bytes;
  15         unsigned recv_msgs;
  16         unsigned write_bytes;
  17         unsigned write_msgs;
  18         unsigned read_bytes;
  19         unsigned read_msgs;
  20 };
  21
  22
  23 /*
  24  * These states are used to signal events between the completion handler
  25  * and the main client or server thread.
  26  *
  27  * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV,
  28  * and RDMA_WRITE_COMPLETE for each ping.
  29  */
  30 enum test_state {
  31         IDLE = 1,
  32         CONNECT_REQUEST,
  33         ADDR_RESOLVED,
  34         ROUTE_RESOLVED,
  35         CONNECTED,
  36         RDMA_READ_ADV,
  37         RDMA_READ_COMPLETE,
  38         RDMA_WRITE_ADV,
  39         RDMA_WRITE_COMPLETE,
  40         ERROR,
  41         CLEANUP
  42 };
  43
  44 struct krping_rdma_info {
  45         uint64_t buf;
  46         uint32_t rkey;
  47         uint32_t size;
  48 };
  49
  50 /*
  51  * Control block struct.
  52  */
  53 struct krping_cb {
  54         int server;                     /* 0 iff client */
  55         struct ib_cq *cq;
  56         struct ib_pd *pd;
  57         struct ib_qp *qp;
  58         struct ib_mr *dma_mr;
  59         int use_dmamr;
  60
  61         struct ib_recv_wr rq_wr;        /* recv work request record */
  62         struct ib_sge recv_sgl;         /* recv single SGE */
  63         struct krping_rdma_info recv_buf;/* malloc'd buffer */
  64         struct ib_mr *recv_mr;
  65
  66         struct ib_send_wr sq_wr;        /* send work requrest record */
  67         struct ib_sge send_sgl;
  68         struct krping_rdma_info send_buf;/* single send buf */
  69         struct ib_mr *send_mr;
  70
  71         struct ib_send_wr rdma_sq_wr;   /* rdma work request record */
  72         struct ib_sge rdma_sgl;         /* rdma single SGE */
  73         char *rdma_buf;                 /* used as rdma sink */
  74         u64  rdma_addr;
  75         struct ib_mr *rdma_mr;
  76
  77         uint32_t remote_rkey;           /* remote guys RKEY */
  78         uint64_t remote_addr;           /* remote guys TO */
  79         uint32_t remote_len;            /* remote guys LEN */
  80
  81         char *start_buf;                /* rdma read src */
  82         u64  start_addr;
  83         struct ib_mr *start_mr;
  84
  85         enum test_state state;          /* used for cond/signalling */
  86         struct mtx lock;
  87         struct krping_stats stats;
  88
  89         uint16_t port;                  /* dst port in NBO */
  90         struct in_addr addr;            /* dst addr in NBO */
  91         char *addr_str;                 /* dst addr string */
  92         int verbose;                    /* verbose logging */
  93         int count;                      /* ping count */
  94         int size;                       /* ping data size */
  95         int validate;                   /* validate ping data */
  96         uint64_t memlimit;              /* limit of the physical memory that
  97                                            can be registered with dma_mr mode */
  98
  99         /* CM stuff */
 100         struct rdma_cm_id *cm_id;       /* connection on client side,*/
 101                                         /* listener on service side. */
 102         struct rdma_cm_id *child_cm_id; /* connection on server side */
 103         TAILQ_ENTRY(krping_cb) list;
 104
 105         int rlat;                       /* run read latency test */
 106         int wlat;                       /* run write latency test */
 107         int bw;                         /* run write bw test */
 108         int duplex;                     /* run write bw full duplex test */
 109         int poll;                       /* poll vs block in rlat */
 110         int txdepth;
 111
 112         char name[16];
 113 };
 114
 115 static __inline uint64_t
 116 get_cycles(void)
 117 {
 118         u_int32_t low, high;
 119         __asm __volatile("rdtsc" : "=a" (low), "=d" (high));
 120         return (low | ((u_int64_t)high << 32));
 121 }
 122
 123 #define htonll(x) htobe64((x))
 124 #define ntohll(x) be64toh((x))
 125
 126 typedef uint64_t cycles_t;
 127
 128 extern struct mtx krping_mutex;
 129 TAILQ_HEAD(krping_cb_list, krping_cb);
 130 extern struct krping_cb_list krping_cbs;
 131
 132 int krping_doit(char *cmd);
 133 void krping_init(void);