sys/contrib/ena-com/ena_plat.h

   1 /*-
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  *
   4  * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * * Redistributions of source code must retain the above copyright
  12  * notice, this list of conditions and the following disclaimer.
  13  * * Redistributions in binary form must reproduce the above copyright
  14  * notice, this list of conditions and the following disclaimer in
  15  * the documentation and/or other materials provided with the
  16  * distribution.
  17  * * Neither the name of copyright holder nor the names of its
  18  * contributors may be used to endorse or promote products derived
  19  * from this software without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #ifndef ENA_PLAT_H_
  35 #define ENA_PLAT_H_
  36
  37 #include <sys/cdefs.h>
  38 __FBSDID("$FreeBSD$");
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42
  43 #include <sys/bus.h>
  44 #include <sys/condvar.h>
  45 #include <sys/endian.h>
  46 #include <sys/kernel.h>
  47 #include <sys/kthread.h>
  48 #include <sys/malloc.h>
  49 #include <sys/mbuf.h>
  50 #include <sys/module.h>
  51 #include <sys/rman.h>
  52 #include <sys/proc.h>
  53 #include <sys/smp.h>
  54 #include <sys/socket.h>
  55 #include <sys/sockio.h>
  56 #include <sys/sysctl.h>
  57 #include <sys/taskqueue.h>
  58 #include <sys/eventhandler.h>
  59 #include <sys/types.h>
  60 #include <sys/timetc.h>
  61 #include <sys/cdefs.h>
  62
  63 #include <machine/atomic.h>
  64 #include <machine/bus.h>
  65 #include <machine/in_cksum.h>
  66 #include <machine/pcpu.h>
  67 #include <machine/resource.h>
  68 #include <machine/_inttypes.h>
  69
  70 #include <net/bpf.h>
  71 #include <net/ethernet.h>
  72 #include <net/if.h>
  73 #include <net/if_var.h>
  74 #include <net/if_arp.h>
  75 #include <net/if_dl.h>
  76 #include <net/if_media.h>
  77
  78 #include <net/if_types.h>
  79 #include <net/if_vlan_var.h>
  80
  81 #include <netinet/in_systm.h>
  82 #include <netinet/in.h>
  83 #include <netinet/if_ether.h>
  84 #include <netinet/ip.h>
  85 #include <netinet/ip6.h>
  86 #include <netinet/tcp.h>
  87 #include <netinet/tcp_lro.h>
  88 #include <netinet/udp.h>
  89
  90 #include <dev/led/led.h>
  91 #include <dev/pci/pcivar.h>
  92 #include <dev/pci/pcireg.h>
  93
  94 extern struct ena_bus_space ebs;
  95
  96 /* Levels */
  97 #define ENA_ALERT       (1 << 0) /* Alerts are providing more error info.     */
  98 #define ENA_WARNING     (1 << 1) /* Driver output is more error sensitive.    */
  99 #define ENA_INFO        (1 << 2) /* Provides additional driver info.          */
 100 #define ENA_DBG         (1 << 3) /* Driver output for debugging.              */
 101 /* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag.        */
 102 #define ENA_TXPTH       (1 << 4) /* Allows TX path tracing.                   */
 103 #define ENA_RXPTH       (1 << 5) /* Allows RX path tracing.                   */
 104 #define ENA_RSC         (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */
 105 #define ENA_IOQ         (1 << 7) /* Detailed info about IO queues.            */
 106 #define ENA_ADMQ        (1 << 8) /* Detailed info about admin queue.          */
 107 #define ENA_NETMAP      (1 << 9) /* Detailed info about netmap.               */
 108
 109 #define DEFAULT_ALLOC_ALIGNMENT 8
 110
 111 extern int ena_log_level;
 112
 113 #define container_of(ptr, type, member)                                 \
 114         ({                                                              \
 115                 const __typeof(((type *)0)->member) *__p = (ptr);       \
 116                 (type *)((uintptr_t)__p - offsetof(type, member));      \
 117         })
 118
 119 #define ena_trace_raw(ctx, level, fmt, args...)                 \
 120         do {                                                    \
 121                 ((void)(ctx));                                  \
 122                 if (((level) & ena_log_level) != (level))       \
 123                         break;                                  \
 124                 printf(fmt, ##args);                            \
 125         } while (0)
 126
 127 #define ena_trace(ctx, level, fmt, args...)                     \
 128         ena_trace_raw(ctx, level, "%s() [TID:%d]: "             \
 129             fmt, __func__, curthread->td_tid, ##args)
 130
 131
 132 #define ena_trc_dbg(ctx, format, arg...)        \
 133         ena_trace(ctx, ENA_DBG, format, ##arg)
 134 #define ena_trc_info(ctx, format, arg...)       \
 135         ena_trace(ctx, ENA_INFO, format, ##arg)
 136 #define ena_trc_warn(ctx, format, arg...)       \
 137         ena_trace(ctx, ENA_WARNING, format, ##arg)
 138 #define ena_trc_err(ctx, format, arg...)        \
 139         ena_trace(ctx, ENA_ALERT, format, ##arg)
 140
 141 #define unlikely(x)     __predict_false(!!(x))
 142 #define likely(x)       __predict_true(!!(x))
 143
 144 #define __iomem
 145 #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE)
 146
 147 #define MAX_ERRNO 4095
 148 #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO)
 149
 150 #define ENA_WARN(cond, ctx, format, arg...)                             \
 151         do {                                                            \
 152                 if (unlikely((cond))) {                                 \
 153                         ena_trc_warn(ctx, format, ##arg);               \
 154                 }                                                       \
 155         } while (0)
 156
 157 static inline long IS_ERR(const void *ptr)
 158 {
 159         return IS_ERR_VALUE((unsigned long)ptr);
 160 }
 161
 162 static inline void *ERR_PTR(long error)
 163 {
 164         return (void *)error;
 165 }
 166
 167 static inline long PTR_ERR(const void *ptr)
 168 {
 169         return (long) ptr;
 170 }
 171
 172 #define GENMASK(h, l)   (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h))))
 173 #define GENMASK_ULL(h, l)       (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h))))
 174 #define BIT(x)                  (1UL << (x))
 175
 176 #define ENA_ABORT()             BUG()
 177 #define BUG()                   panic("ENA BUG")
 178
 179 #define SZ_256                  (256)
 180 #define SZ_4K                   (4096)
 181
 182 #define ENA_COM_OK              0
 183 #define ENA_COM_FAULT           EFAULT
 184 #define ENA_COM_INVAL           EINVAL
 185 #define ENA_COM_NO_MEM          ENOMEM
 186 #define ENA_COM_NO_SPACE        ENOSPC
 187 #define ENA_COM_TRY_AGAIN       -1
 188 #define ENA_COM_UNSUPPORTED     EOPNOTSUPP
 189 #define ENA_COM_NO_DEVICE       ENODEV
 190 #define ENA_COM_PERMISSION      EPERM
 191 #define ENA_COM_TIMER_EXPIRED   ETIMEDOUT
 192 #define ENA_COM_EIO             EIO
 193
 194 #define ENA_MSLEEP(x)           pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0)
 195 #define ENA_USLEEP(x)           pause_sbt("ena", SBT_1US * (x), SBT_1US, 0)
 196 #define ENA_UDELAY(x)           DELAY(x)
 197 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \
 198     ((long)cputick2usec(cpu_ticks()) + (timeout_us))
 199 #define ENA_TIME_EXPIRE(timeout)  ((timeout) < cputick2usec(cpu_ticks()))
 200 #define ENA_MIGHT_SLEEP()
 201
 202 #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y))
 203 #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y))
 204
 205 #define ENA_MIN32(x,y)  MIN(x, y)
 206 #define ENA_MIN16(x,y)  MIN(x, y)
 207 #define ENA_MIN8(x,y)   MIN(x, y)
 208
 209 #define ENA_MAX32(x,y)  MAX(x, y)
 210 #define ENA_MAX16(x,y)  MAX(x, y)
 211 #define ENA_MAX8(x,y)   MAX(x, y)
 212
 213 /* Spinlock related methods */
 214 #define ena_spinlock_t  struct mtx
 215 #define ENA_SPINLOCK_INIT(spinlock)                             \
 216         mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN)
 217 #define ENA_SPINLOCK_DESTROY(spinlock)                          \
 218         do {                                                    \
 219                 if (mtx_initialized(&(spinlock)))               \
 220                     mtx_destroy(&(spinlock));                   \
 221         } while (0)
 222 #define ENA_SPINLOCK_LOCK(spinlock, flags)                      \
 223         do {                                                    \
 224                 (void)(flags);                                  \
 225                 mtx_lock_spin(&(spinlock));                     \
 226         } while (0)
 227 #define ENA_SPINLOCK_UNLOCK(spinlock, flags)                    \
 228         do {                                                    \
 229                 (void)(flags);                                  \
 230                 mtx_unlock_spin(&(spinlock));                   \
 231         } while (0)
 232
 233
 234 /* Wait queue related methods */
 235 #define ena_wait_event_t struct { struct cv wq; struct mtx mtx; }
 236 #define ENA_WAIT_EVENT_INIT(waitqueue)                                  \
 237         do {                                                            \
 238                 cv_init(&((waitqueue).wq), "cv");                       \
 239                 mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF);      \
 240         } while (0)
 241 #define ENA_WAIT_EVENTS_DESTROY(admin_queue)                            \
 242         do {                                                            \
 243                 struct ena_comp_ctx *comp_ctx;                          \
 244                 int i;                                                  \
 245                 for (i = 0; i < admin_queue->q_depth; i++) {            \
 246                         comp_ctx = get_comp_ctxt(admin_queue, i, false); \
 247                         if (comp_ctx != NULL) {                         \
 248                                 cv_destroy(&((comp_ctx->wait_event).wq)); \
 249                                 mtx_destroy(&((comp_ctx->wait_event).mtx)); \
 250                         }                                               \
 251                 }                                                       \
 252         } while (0)
 253 #define ENA_WAIT_EVENT_CLEAR(waitqueue)                                 \
 254         cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description)
 255 #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us)                      \
 256         do {                                                            \
 257                 mtx_lock(&((waitqueue).mtx));                           \
 258                 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx),     \
 259                     timeout_us * hz / 1000 / 1000 );                    \
 260                 mtx_unlock(&((waitqueue).mtx));                         \
 261         } while (0)
 262 #define ENA_WAIT_EVENT_SIGNAL(waitqueue)                \
 263         do {                                            \
 264                 mtx_lock(&((waitqueue).mtx));           \
 265                 cv_broadcast(&((waitqueue).wq));        \
 266                 mtx_unlock(&((waitqueue).mtx));         \
 267         } while (0)
 268
 269 #define dma_addr_t      bus_addr_t
 270 #define u8              uint8_t
 271 #define u16             uint16_t
 272 #define u32             uint32_t
 273 #define u64             uint64_t
 274
 275 typedef struct {
 276         bus_addr_t              paddr;
 277         caddr_t                 vaddr;
 278         bus_dma_tag_t           tag;
 279         bus_dmamap_t            map;
 280         bus_dma_segment_t       seg;
 281         int                     nseg;
 282 } ena_mem_handle_t;
 283
 284 struct ena_bus {
 285         bus_space_handle_t      reg_bar_h;
 286         bus_space_tag_t         reg_bar_t;
 287         bus_space_handle_t      mem_bar_h;
 288         bus_space_tag_t         mem_bar_t;
 289 };
 290
 291 typedef uint32_t ena_atomic32_t;
 292
 293 #define ENA_PRIu64 PRIu64
 294
 295 typedef uint64_t ena_time_t;
 296 typedef struct ifnet ena_netdev;
 297
 298 void    ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg,
 299     int error);
 300 int     ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
 301     int mapflags, bus_size_t alignment);
 302
 303 static inline uint32_t
 304 ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
 305 {
 306         uint32_t v = bus_space_read_4(bus->reg_bar_t, bus->reg_bar_h, offset);
 307         rmb();
 308         return v;
 309 }
 310
 311 #define ENA_MEMCPY_TO_DEVICE_64(dst, src, size)                         \
 312         do {                                                            \
 313                 int count, i;                                           \
 314                 volatile uint64_t *to = (volatile uint64_t *)(dst);     \
 315                 const uint64_t *from = (const uint64_t *)(src);         \
 316                 count = (size) / 8;                                     \
 317                                                                         \
 318                 for (i = 0; i < count; i++, from++, to++)               \
 319                         *to = *from;                                    \
 320         } while (0)
 321
 322 #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO)
 323 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL)
 324 #define ENA_MEM_FREE(dmadev, ptr, size)                                 \
 325         do {                                                            \
 326                 (void)(size);                                           \
 327                 free(ptr, M_DEVBUF);                                    \
 328         } while (0)
 329 #define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys,   \
 330     handle, node, dev_node, alignment)                                  \
 331         do {                                                            \
 332                 ((virt) = NULL);                                        \
 333                 (void)(dev_node);                                       \
 334         } while (0)
 335
 336 #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle,   \
 337     node, dev_node)                                                     \
 338         ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt,         \
 339             phys, handle, node, dev_node, DEFAULT_ALLOC_ALIGNMENT)
 340
 341 #define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma,   \
 342     alignment)                                                          \
 343         do {                                                            \
 344                 ena_dma_alloc((dmadev), (size), &(dma), 0, alignment);  \
 345                 (virt) = (void *)(dma).vaddr;                           \
 346                 (phys) = (dma).paddr;                                   \
 347         } while (0)
 348
 349 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma)           \
 350         ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt,              \
 351             phys, dma, DEFAULT_ALLOC_ALIGNMENT)
 352
 353 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma)            \
 354         do {                                                            \
 355                 (void)size;                                             \
 356                 bus_dmamap_unload((dma).tag, (dma).map);                \
 357                 bus_dmamem_free((dma).tag, (virt), (dma).map);          \
 358                 bus_dma_tag_destroy((dma).tag);                         \
 359                 (dma).tag = NULL;                                       \
 360                 (virt) = NULL;                                          \
 361         } while (0)
 362
 363 /* Register R/W methods */
 364 #define ENA_REG_WRITE32(bus, value, offset)                             \
 365         do {                                                            \
 366                 wmb();                                                  \
 367                 ENA_REG_WRITE32_RELAXED(bus, value, offset);            \
 368         } while (0)
 369
 370 #define ENA_REG_WRITE32_RELAXED(bus, value, offset)                     \
 371         bus_space_write_4(                                              \
 372                           ((struct ena_bus*)bus)->reg_bar_t,            \
 373                           ((struct ena_bus*)bus)->reg_bar_h,            \
 374                           (bus_size_t)(offset), (value))
 375
 376 #define ENA_REG_READ32(bus, offset)                                     \
 377         ena_reg_read32((struct ena_bus*)(bus), (bus_size_t)(offset))
 378
 379 #define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync(                  \
 380         (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE)
 381 #define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync(                \
 382         (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD)
 383 #define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync(               \
 384         (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD)
 385 #define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle)
 386
 387 #define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0)
 388
 389 #define VLAN_HLEN       sizeof(struct ether_vlan_header)
 390 #define CSUM_OFFLOAD    (CSUM_IP|CSUM_TCP|CSUM_UDP)
 391
 392 #define prefetch(x)     (void)(x)
 393 #define prefetchw(x)    (void)(x)
 394
 395 /* DMA buffers access */
 396 #define dma_unmap_addr(p, name)                 ((p)->dma->name)
 397 #define dma_unmap_addr_set(p, name, v)          (((p)->dma->name) = (v))
 398 #define dma_unmap_len(p, name)                  ((p)->name)
 399 #define dma_unmap_len_set(p, name, v)           (((p)->name) = (v))
 400
 401 #define memcpy_toio memcpy
 402
 403 #define ATOMIC32_INC(I32_PTR)           atomic_add_int(I32_PTR, 1)
 404 #define ATOMIC32_DEC(I32_PTR)           atomic_add_int(I32_PTR, -1)
 405 #define ATOMIC32_READ(I32_PTR)          atomic_load_acq_int(I32_PTR)
 406 #define ATOMIC32_SET(I32_PTR, VAL)      atomic_store_rel_int(I32_PTR, VAL)
 407
 408 #define barrier() __asm__ __volatile__("": : :"memory")
 409 #define dma_rmb() barrier()
 410 #define mmiowb() barrier()
 411
 412 #define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x))
 413 #define READ_ONCE(x)  ({                        \
 414                         __typeof(x) __var;      \
 415                         barrier();              \
 416                         __var = ACCESS_ONCE(x); \
 417                         barrier();              \
 418                         __var;                  \
 419                 })
 420 #define READ_ONCE8(x) READ_ONCE(x)
 421 #define READ_ONCE16(x) READ_ONCE(x)
 422 #define READ_ONCE32(x) READ_ONCE(x)
 423
 424 #define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16))
 425 #define lower_32_bits(n) ((uint32_t)(n))
 426
 427 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
 428
 429 #define ENA_FFS(x) ffs(x)
 430
 431 void    ena_rss_key_fill(void *key, size_t size);
 432
 433 #define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size)
 434
 435 #include "ena_defs/ena_includes.h"
 436
 437 #endif /* ENA_PLAT_H_ */