2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2021 Microsoft Corp.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include <sys/bus_dma.h>
39 #include <sys/types.h>
40 #include <sys/limits.h>
43 #include "gdma_util.h"
44 #include "shm_channel.h"
46 /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
47 * them are naturally aligned and hence don't need __packed.
52 #define GDMA_IRQNAME_SZ 40
55 bus_space_handle_t bar0_h;
56 bus_space_tag_t bar0_t;
59 struct gdma_msix_entry {
64 enum gdma_request_type {
65 GDMA_VERIFY_VF_DRIVER_VERSION = 1,
66 GDMA_QUERY_MAX_RESOURCES = 2,
67 GDMA_LIST_DEVICES = 3,
68 GDMA_REGISTER_DEVICE = 4,
69 GDMA_DEREGISTER_DEVICE = 5,
70 GDMA_GENERATE_TEST_EQE = 10,
71 GDMA_CREATE_QUEUE = 12,
72 GDMA_DISABLE_QUEUE = 13,
73 GDMA_CREATE_DMA_REGION = 25,
74 GDMA_DMA_REGION_ADD_PAGES = 26,
75 GDMA_DESTROY_DMA_REGION = 27,
78 enum gdma_queue_type {
86 enum gdma_work_request_flags {
88 GDMA_WR_OOB_IN_SGL = BIT(0),
89 GDMA_WR_PAD_BY_SGE0 = BIT(1),
93 GDMA_EQE_COMPLETION = 3,
94 GDMA_EQE_TEST_EVENT = 64,
95 GDMA_EQE_HWC_INIT_EQ_ID_DB = 129,
96 GDMA_EQE_HWC_INIT_DATA = 130,
97 GDMA_EQE_HWC_INIT_DONE = 131,
101 GDMA_DEVICE_NONE = 0,
103 GDMA_DEVICE_MANA = 2,
107 struct gdma_resource {
108 /* Protect the bitmap */
109 struct mtx lock_spin;
111 /* The bitmap size in bits. */
114 /* The bitmap tracks the resources. */
118 union gdma_doorbell_entry {
123 uint64_t reserved : 8;
124 uint64_t tail_ptr : 31;
130 uint64_t wqe_cnt : 8;
131 uint64_t tail_ptr : 32;
136 uint64_t reserved : 8;
137 uint64_t tail_ptr : 32;
142 uint64_t reserved : 16;
143 uint64_t tail_ptr : 31;
148 struct gdma_msg_hdr {
151 uint16_t msg_version;
167 struct gdma_req_hdr {
168 struct gdma_msg_hdr req;
169 struct gdma_msg_hdr resp; /* The expected response */
170 struct gdma_dev_id dev_id;
171 uint32_t activity_id;
174 struct gdma_resp_hdr {
175 struct gdma_msg_hdr response;
176 struct gdma_dev_id dev_id;
177 uint32_t activity_id;
182 struct gdma_general_req {
183 struct gdma_req_hdr hdr;
186 #define GDMA_MESSAGE_V1 1
188 struct gdma_general_resp {
189 struct gdma_resp_hdr hdr;
192 #define GDMA_STANDARD_HEADER_TYPE 0
195 mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, uint32_t code,
196 uint32_t req_size, uint32_t resp_size)
198 hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
199 hdr->req.msg_type = code;
200 hdr->req.msg_version = GDMA_MESSAGE_V1;
201 hdr->req.msg_size = req_size;
203 hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
204 hdr->resp.msg_type = code;
205 hdr->resp.msg_version = GDMA_MESSAGE_V1;
206 hdr->resp.msg_size = resp_size;
209 /* The 16-byte struct is part of the GDMA work queue entry (WQE). */
216 struct gdma_wqe_request {
217 struct gdma_sge *sgl;
220 uint32_t inline_oob_size;
221 const void *inline_oob_data;
224 uint32_t client_data_unit;
227 enum gdma_page_type {
231 #define GDMA_INVALID_DMA_REGION 0
233 struct gdma_mem_info {
236 bus_dma_tag_t dma_tag;
237 bus_dmamap_t dma_map;
238 bus_addr_t dma_handle; /* Physical address */
239 void *virt_addr; /* Virtual address */
242 /* Allocated by the PF driver */
243 uint64_t gdma_region;
246 #define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
249 struct gdma_context *gdma_context;
251 struct gdma_dev_id dev_id;
257 /* GDMA driver specific pointer */
261 #define MINIMUM_SUPPORTED_PAGE_SIZE PAGE_SIZE
263 #define GDMA_CQE_SIZE 64
264 #define GDMA_EQE_SIZE 16
265 #define GDMA_MAX_SQE_SIZE 512
266 #define GDMA_MAX_RQE_SIZE 256
268 #define GDMA_COMP_DATA_SIZE 0x3C
270 #define GDMA_EVENT_DATA_SIZE 0xC
272 /* The WQE size must be a multiple of the Basic Unit, which is 32 bytes. */
273 #define GDMA_WQE_BU_SIZE 32
275 #define INVALID_PDID UINT_MAX
276 #define INVALID_DOORBELL UINT_MAX
277 #define INVALID_MEM_KEY UINT_MAX
278 #define INVALID_QUEUE_ID UINT_MAX
279 #define INVALID_PCI_MSIX_INDEX UINT_MAX
282 uint32_t cqe_data[GDMA_COMP_DATA_SIZE / 4];
288 uint32_t details[GDMA_EVENT_DATA_SIZE / 4];
294 typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
295 struct gdma_event *e);
297 typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
299 /* The 'head' is the producer index. For SQ/RQ, when the driver posts a WQE
300 * (Note: the WQE size must be a multiple of the 32-byte Basic Unit), the
301 * driver increases the 'head' in BUs rather than in bytes, and notifies
302 * the HW of the updated head. For EQ/CQ, the driver uses the 'head' to track
303 * the HW head, and increases the 'head' by 1 for every processed EQE/CQE.
305 * The 'tail' is the consumer index for SQ/RQ. After the CQE of the SQ/RQ is
306 * processed, the driver increases the 'tail' to indicate that WQEs have
307 * been consumed by the HW, so the driver can post new WQEs into the SQ/RQ.
309 * The driver doesn't use the 'tail' for EQ/CQ, because the driver ensures
310 * that the EQ/CQ is big enough so they can't overflow, and the driver uses
311 * the owner bits mechanism to detect if the queue has become empty.
314 struct gdma_dev *gdma_dev;
316 enum gdma_queue_type type;
319 struct gdma_mem_info mem_info;
324 bool monitor_avl_buf;
329 /* Extra fields specific to EQ/CQ. */
334 gdma_eq_callback *callback;
337 unsigned int msix_index;
339 uint32_t log2_throttle_limit;
343 gdma_cq_callback *callback;
346 /* For CQ/EQ relationship */
347 struct gdma_queue *parent;
352 struct gdma_queue_spec {
353 enum gdma_queue_type type;
354 bool monitor_avl_buf;
355 unsigned int queue_size;
357 /* Extra fields specific to EQ/CQ. */
360 gdma_eq_callback *callback;
363 unsigned long log2_throttle_limit;
367 gdma_cq_callback *callback;
370 struct gdma_queue *parent_eq;
377 struct gdma_queue *eq;
380 struct gdma_irq_context {
381 struct gdma_msix_entry msix_e;
382 struct resource *res;
383 driver_intr_t *handler;
388 char name[GDMA_IRQNAME_SZ];
391 struct gdma_context {
394 struct gdma_bus gd_bus;
396 /* Per-vPort max number of queues */
397 unsigned int max_num_queues;
398 unsigned int max_num_msix;
399 unsigned int num_msix_usable;
400 struct gdma_resource msix_resource;
401 struct gdma_irq_context *irq_contexts;
403 /* This maps a CQ index to the queue structure. */
404 unsigned int max_num_cqs;
405 struct gdma_queue **cq_table;
407 /* Protect eq_test_event and test_event_eq_id */
408 struct sx eq_test_event_sx;
409 struct completion eq_test_event;
410 uint32_t test_event_eq_id;
412 struct resource *bar0;
413 struct resource *msix;
415 void __iomem *shm_base;
416 void __iomem *db_page_base;
417 uint32_t db_page_size;
419 /* Shared memory chanenl (used to bootstrap HWC) */
420 struct shm_channel shm_channel;
422 /* Hardware communication channel (HWC) */
425 /* Azure network adapter */
426 struct gdma_dev mana;
429 #define MAX_NUM_GDMA_DEVICES 4
431 static inline bool mana_gd_is_mana(struct gdma_dev *gd)
433 return gd->dev_id.type == GDMA_DEVICE_MANA;
436 static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
438 return gd->dev_id.type == GDMA_DEVICE_HWC;
441 uint8_t *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, uint32_t wqe_offset);
442 uint32_t mana_gd_wq_avail_space(struct gdma_queue *wq);
444 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
446 int mana_gd_create_hwc_queue(struct gdma_dev *gd,
447 const struct gdma_queue_spec *spec,
448 struct gdma_queue **queue_ptr);
450 int mana_gd_create_mana_eq(struct gdma_dev *gd,
451 const struct gdma_queue_spec *spec,
452 struct gdma_queue **queue_ptr);
454 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
455 const struct gdma_queue_spec *spec,
456 struct gdma_queue **queue_ptr);
458 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
460 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
462 void mana_gd_ring_cq(struct gdma_queue *cq, uint8_t arm_bit);
465 uint32_t reserved :24;
466 uint32_t last_vbytes :8;
473 uint32_t inline_oob_size_div4 :3;
474 uint32_t client_oob_in_sgl :1;
475 uint32_t reserved1 :4;
476 uint32_t client_data_unit :14;
477 uint32_t reserved2 :2;
482 #define INLINE_OOB_SMALL_SIZE 8
483 #define INLINE_OOB_LARGE_SIZE 24
485 #define MAX_TX_WQE_SIZE 512
486 #define MAX_RX_WQE_SIZE 256
489 uint32_t cqe_data[GDMA_COMP_DATA_SIZE / 4];
497 uint32_t reserved :4;
498 uint32_t owner_bits :3;
503 #define GDMA_CQE_OWNER_BITS 3
505 #define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
507 #define SET_ARM_BIT 1
509 #define GDMA_EQE_OWNER_BITS 3
511 union gdma_eqe_info {
516 uint32_t reserved1 : 8;
517 uint32_t client_id : 2;
518 uint32_t reserved2 : 11;
519 uint32_t owner_bits : 3;
523 #define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
524 #define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
527 uint32_t details[GDMA_EVENT_DATA_SIZE / 4];
531 #define GDMA_REG_DB_PAGE_OFFSET 8
532 #define GDMA_REG_DB_PAGE_SIZE 0x10
533 #define GDMA_REG_SHM_OFFSET 0x18
535 struct gdma_posted_wqe_info {
536 uint32_t wqe_size_in_bu;
539 /* GDMA_GENERATE_TEST_EQE */
540 struct gdma_generate_test_event_req {
541 struct gdma_req_hdr hdr;
542 uint32_t queue_index;
545 /* GDMA_VERIFY_VF_DRIVER_VERSION */
547 GDMA_PROTOCOL_V1 = 1,
548 GDMA_PROTOCOL_FIRST = GDMA_PROTOCOL_V1,
549 GDMA_PROTOCOL_LAST = GDMA_PROTOCOL_V1,
552 struct gdma_verify_ver_req {
553 struct gdma_req_hdr hdr;
555 /* Mandatory fields required for protocol establishment */
556 uint64_t protocol_ver_min;
557 uint64_t protocol_ver_max;
558 uint64_t drv_cap_flags1;
559 uint64_t drv_cap_flags2;
560 uint64_t drv_cap_flags3;
561 uint64_t drv_cap_flags4;
563 /* Advisory fields */
565 uint32_t os_type; /* Linux = 0x10; Windows = 0x20; Other = 0x30 */
567 uint32_t os_ver_major;
568 uint32_t os_ver_minor;
569 uint32_t os_ver_build;
570 uint32_t os_ver_platform;
572 uint8_t os_ver_str1[128];
573 uint8_t os_ver_str2[128];
574 uint8_t os_ver_str3[128];
575 uint8_t os_ver_str4[128];
578 struct gdma_verify_ver_resp {
579 struct gdma_resp_hdr hdr;
580 uint64_t gdma_protocol_ver;
581 uint64_t pf_cap_flags1;
582 uint64_t pf_cap_flags2;
583 uint64_t pf_cap_flags3;
584 uint64_t pf_cap_flags4;
587 /* GDMA_QUERY_MAX_RESOURCES */
588 struct gdma_query_max_resources_resp {
589 struct gdma_resp_hdr hdr;
597 uint32_t max_cq_mod_ctx;
602 /* GDMA_LIST_DEVICES */
603 struct gdma_list_devices_resp {
604 struct gdma_resp_hdr hdr;
605 uint32_t num_of_devs;
607 struct gdma_dev_id devs[64];
610 /* GDMA_REGISTER_DEVICE */
611 struct gdma_register_device_resp {
612 struct gdma_resp_hdr hdr;
618 /* GDMA_CREATE_QUEUE */
619 struct gdma_create_queue_req {
620 struct gdma_req_hdr hdr;
624 uint32_t doolbell_id;
625 uint64_t gdma_region;
628 uint32_t log2_throttle_limit;
629 uint32_t eq_pci_msix_index;
630 uint32_t cq_mod_ctx_id;
631 uint32_t cq_parent_eq_id;
632 uint8_t rq_drop_on_overrun;
633 uint8_t rq_err_on_wqe_overflow;
634 uint8_t rq_chain_rec_wqes;
639 struct gdma_create_queue_resp {
640 struct gdma_resp_hdr hdr;
641 uint32_t queue_index;
644 /* GDMA_DISABLE_QUEUE */
645 struct gdma_disable_queue_req {
646 struct gdma_req_hdr hdr;
648 uint32_t queue_index;
649 uint32_t alloc_res_id_on_creation;
652 /* GDMA_CREATE_DMA_REGION */
653 struct gdma_create_dma_region_req {
654 struct gdma_req_hdr hdr;
656 /* The total size of the DMA region */
659 /* The offset in the first page */
660 uint32_t offset_in_page;
662 /* enum gdma_page_type */
663 uint32_t gdma_page_type;
665 /* The total number of pages */
668 /* If page_addr_list_len is smaller than page_count,
669 * the remaining page addresses will be added via the
670 * message GDMA_DMA_REGION_ADD_PAGES.
672 uint32_t page_addr_list_len;
673 uint64_t page_addr_list[];
676 struct gdma_create_dma_region_resp {
677 struct gdma_resp_hdr hdr;
678 uint64_t gdma_region;
681 /* GDMA_DMA_REGION_ADD_PAGES */
682 struct gdma_dma_region_add_pages_req {
683 struct gdma_req_hdr hdr;
685 uint64_t gdma_region;
687 uint32_t page_addr_list_len;
690 uint64_t page_addr_list[];
693 /* GDMA_DESTROY_DMA_REGION */
694 struct gdma_destroy_dma_region_req {
695 struct gdma_req_hdr hdr;
697 uint64_t gdma_region;
700 int mana_gd_verify_vf_version(device_t dev);
702 int mana_gd_register_device(struct gdma_dev *gd);
703 int mana_gd_deregister_device(struct gdma_dev *gd);
705 int mana_gd_post_work_request(struct gdma_queue *wq,
706 const struct gdma_wqe_request *wqe_req,
707 struct gdma_posted_wqe_info *wqe_info);
709 int mana_gd_post_and_ring(struct gdma_queue *queue,
710 const struct gdma_wqe_request *wqe,
711 struct gdma_posted_wqe_info *wqe_info);
713 int mana_gd_alloc_res_map(uint32_t res_avil, struct gdma_resource *r,
714 const char *lock_name);
715 void mana_gd_free_res_map(struct gdma_resource *r);
717 void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
718 struct gdma_queue *queue);
720 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
721 struct gdma_mem_info *gmi);
723 void mana_gd_free_memory(struct gdma_mem_info *gmi);
725 void mana_gd_dma_map_paddr(void *arg, bus_dma_segment_t *segs,
726 int nseg, int error);
728 int mana_gd_send_request(struct gdma_context *gc, uint32_t req_len,
729 const void *req, uint32_t resp_len, void *resp);