2 * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org>
3 * Copyright (C) 2013 Intel Corporation
4 * Copyright (C) 2015 EMC Corporation
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * The Non-Transparent Bridge (NTB) is a device that allows you to connect
31 * two or more systems using a PCI-e links, providing remote memory access.
33 * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs.
35 * NOTE: Much of the code in this module is shared with Linux. Any patches may
36 * be picked up and redistributed in Linux with a dual GPL/BSD license.
39 #include <sys/cdefs.h>
40 __FBSDID("$FreeBSD$");
42 #include <sys/param.h>
43 #include <sys/kernel.h>
44 #include <sys/systm.h>
46 #include <sys/endian.h>
47 #include <sys/interrupt.h>
49 #include <sys/malloc.h>
50 #include <sys/module.h>
51 #include <sys/mutex.h>
52 #include <sys/pciio.h>
53 #include <sys/queue.h>
56 #include <sys/sysctl.h>
59 #include <machine/bus.h>
60 #include <machine/intr_machdep.h>
61 #include <machine/resource.h>
62 #include <dev/pci/pcireg.h>
63 #include <dev/pci/pcivar.h>
65 #include "ntb_hw_intel.h"
68 #define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT)
70 #define NTB_HB_TIMEOUT 1 /* second */
71 #define ATOM_LINK_RECOVERY_TIME 500 /* ms */
72 #define BAR_HIGH_MASK (~((1ull << 12) - 1))
74 #define NTB_MSIX_VER_GUARD 0xaabbccdd
75 #define NTB_MSIX_RECEIVED 0xe0f0e0f0
78 * PCI constants could be somewhere more generic, but aren't defined/used in
81 #define PCI_MSIX_ENTRY_SIZE 16
82 #define PCI_MSIX_ENTRY_LOWER_ADDR 0
83 #define PCI_MSIX_ENTRY_UPPER_ADDR 4
84 #define PCI_MSIX_ENTRY_DATA 8
86 enum ntb_device_type {
91 /* ntb_conn_type are hardware numbers, cannot change. */
93 NTB_CONN_TRANSPARENT = 0,
98 enum ntb_b2b_direction {
123 /* Device features and workarounds */
124 #define HAS_FEATURE(ntb, feature) \
125 (((ntb)->features & (feature)) != 0)
130 enum ntb_device_type type;
134 struct ntb_pci_bar_info {
135 bus_space_tag_t pci_bus_tag;
136 bus_space_handle_t pci_bus_handle;
138 struct resource *pci_resource;
142 vm_memattr_t map_mode;
144 /* Configuration register offsets */
147 uint32_t pbarxlat_off;
150 struct ntb_int_info {
151 struct resource *res;
157 struct ntb_softc *ntb;
166 unsigned mw_bar[NTB_MAX_BARS];
175 struct ntb_xlat_reg {
190 struct ntb_b2b_addr {
192 uint64_t bar2_addr64;
193 uint64_t bar4_addr64;
194 uint64_t bar4_addr32;
195 uint64_t bar5_addr32;
198 struct ntb_msix_data {
204 /* ntb.c context. Do not move! Must go first! */
208 enum ntb_device_type type;
211 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS];
212 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS];
213 uint32_t allocated_interrupts;
215 struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS];
216 struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS];
219 struct ntb_pci_bar_info *peer_lapic_bar;
220 struct callout peer_msix_work;
222 bus_dma_tag_t bar0_dma_tag;
223 bus_dmamap_t bar0_dma_map;
225 struct callout heartbeat_timer;
226 struct callout lr_timer;
228 struct ntb_vec *msix_vec;
231 enum ntb_conn_type conn_type;
232 enum ntb_b2b_direction dev_type;
234 /* Offset of peer bar0 in B2B BAR */
236 /* Memory window used to access peer bar0 */
237 #define B2B_MW_DISABLED UINT8_MAX
245 uint8_t db_vec_count;
246 uint8_t db_vec_shift;
248 /* Protects local db_mask. */
249 #define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock)
250 #define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock)
251 #define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f))
252 struct mtx db_mask_lock;
254 volatile uint32_t ntb_ctl;
255 volatile uint32_t lnk_sta;
257 uint64_t db_valid_mask;
258 uint64_t db_link_mask;
260 uint64_t fake_db; /* NTB_SB01BASE_LOCKUP*/
261 uint64_t force_db; /* NTB_SB01BASE_LOCKUP*/
263 int last_ts; /* ticks @ last irq */
265 const struct ntb_reg *reg;
266 const struct ntb_alt_reg *self_reg;
267 const struct ntb_alt_reg *peer_reg;
268 const struct ntb_xlat_reg *xlat_reg;
272 static __inline uint64_t
273 bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
277 return (bus_space_read_4(tag, handle, offset) |
278 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32);
282 bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle,
283 bus_size_t offset, uint64_t val)
286 bus_space_write_4(tag, handle, offset, val);
287 bus_space_write_4(tag, handle, offset + 4, val >> 32);
291 #define intel_ntb_bar_read(SIZE, bar, offset) \
292 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
293 ntb->bar_info[(bar)].pci_bus_handle, (offset))
294 #define intel_ntb_bar_write(SIZE, bar, offset, val) \
295 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \
296 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val))
297 #define intel_ntb_reg_read(SIZE, offset) \
298 intel_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset)
299 #define intel_ntb_reg_write(SIZE, offset, val) \
300 intel_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val)
301 #define intel_ntb_mw_read(SIZE, offset) \
302 intel_ntb_bar_read(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \
304 #define intel_ntb_mw_write(SIZE, offset, val) \
305 intel_ntb_bar_write(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \
308 static int intel_ntb_probe(device_t device);
309 static int intel_ntb_attach(device_t device);
310 static int intel_ntb_detach(device_t device);
311 static uint64_t intel_ntb_db_valid_mask(device_t dev);
312 static void intel_ntb_spad_clear(device_t dev);
313 static uint64_t intel_ntb_db_vector_mask(device_t dev, uint32_t vector);
314 static bool intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed,
315 enum ntb_width *width);
316 static int intel_ntb_link_enable(device_t dev, enum ntb_speed speed,
317 enum ntb_width width);
318 static int intel_ntb_link_disable(device_t dev);
319 static int intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val);
320 static int intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val);
322 static unsigned intel_ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx);
323 static inline enum ntb_bar intel_ntb_mw_to_bar(struct ntb_softc *, unsigned mw);
324 static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar);
325 static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar,
326 uint32_t *base, uint32_t *xlat, uint32_t *lmt);
327 static int intel_ntb_map_pci_bars(struct ntb_softc *ntb);
328 static int intel_ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx,
330 static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *,
332 static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar);
333 static int map_memory_window_bar(struct ntb_softc *ntb,
334 struct ntb_pci_bar_info *bar);
335 static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb);
336 static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
337 static int intel_ntb_init_isr(struct ntb_softc *ntb);
338 static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
339 static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
340 static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb);
341 static inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector);
342 static void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec);
343 static void ndev_vec_isr(void *arg);
344 static void ndev_irq_isr(void *arg);
345 static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff);
346 static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t);
347 static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t);
348 static int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors);
349 static void intel_ntb_free_msix_vec(struct ntb_softc *ntb);
350 static void intel_ntb_get_msix_info(struct ntb_softc *ntb);
351 static void intel_ntb_exchange_msix(void *);
352 static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id);
353 static void intel_ntb_detect_max_mw(struct ntb_softc *ntb);
354 static int intel_ntb_detect_xeon(struct ntb_softc *ntb);
355 static int intel_ntb_detect_atom(struct ntb_softc *ntb);
356 static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb);
357 static int intel_ntb_atom_init_dev(struct ntb_softc *ntb);
358 static void intel_ntb_teardown_xeon(struct ntb_softc *ntb);
359 static void configure_atom_secondary_side_bars(struct ntb_softc *ntb);
360 static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx,
361 enum ntb_bar regbar);
362 static void xeon_set_sbar_base_and_limit(struct ntb_softc *,
363 uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar);
364 static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
366 static int xeon_setup_b2b_mw(struct ntb_softc *,
367 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
368 static inline bool link_is_up(struct ntb_softc *ntb);
369 static inline bool _xeon_link_is_up(struct ntb_softc *ntb);
370 static inline bool atom_link_is_err(struct ntb_softc *ntb);
371 static inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *);
372 static inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *);
373 static void atom_link_hb(void *arg);
374 static void recover_atom_link(void *arg);
375 static bool intel_ntb_poll_link(struct ntb_softc *ntb);
376 static void save_bar_parameters(struct ntb_pci_bar_info *bar);
377 static void intel_ntb_sysctl_init(struct ntb_softc *);
378 static int sysctl_handle_features(SYSCTL_HANDLER_ARGS);
379 static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS);
380 static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS);
381 static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS);
382 static int sysctl_handle_register(SYSCTL_HANDLER_ARGS);
384 static unsigned g_ntb_hw_debug_level;
385 SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN,
386 &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose");
387 #define intel_ntb_printf(lvl, ...) do { \
388 if ((lvl) <= g_ntb_hw_debug_level) { \
389 device_printf(ntb->device, __VA_ARGS__); \
393 #define _NTB_PAT_UC 0
394 #define _NTB_PAT_WC 1
395 #define _NTB_PAT_WT 4
396 #define _NTB_PAT_WP 5
397 #define _NTB_PAT_WB 6
398 #define _NTB_PAT_UCM 7
399 static unsigned g_ntb_mw_pat = _NTB_PAT_UC;
400 SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN,
401 &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): "
402 "UC: " __XSTRING(_NTB_PAT_UC) ", "
403 "WC: " __XSTRING(_NTB_PAT_WC) ", "
404 "WT: " __XSTRING(_NTB_PAT_WT) ", "
405 "WP: " __XSTRING(_NTB_PAT_WP) ", "
406 "WB: " __XSTRING(_NTB_PAT_WB) ", "
407 "UC-: " __XSTRING(_NTB_PAT_UCM));
409 static inline vm_memattr_t
410 intel_ntb_pat_flags(void)
413 switch (g_ntb_mw_pat) {
415 return (VM_MEMATTR_WRITE_COMBINING);
417 return (VM_MEMATTR_WRITE_THROUGH);
419 return (VM_MEMATTR_WRITE_PROTECTED);
421 return (VM_MEMATTR_WRITE_BACK);
423 return (VM_MEMATTR_WEAK_UNCACHEABLE);
427 return (VM_MEMATTR_UNCACHEABLE);
432 * Well, this obviously doesn't belong here, but it doesn't seem to exist
433 * anywhere better yet.
435 static inline const char *
436 intel_ntb_vm_memattr_to_str(vm_memattr_t pat)
440 case VM_MEMATTR_WRITE_COMBINING:
441 return ("WRITE_COMBINING");
442 case VM_MEMATTR_WRITE_THROUGH:
443 return ("WRITE_THROUGH");
444 case VM_MEMATTR_WRITE_PROTECTED:
445 return ("WRITE_PROTECTED");
446 case VM_MEMATTR_WRITE_BACK:
447 return ("WRITE_BACK");
448 case VM_MEMATTR_WEAK_UNCACHEABLE:
450 case VM_MEMATTR_UNCACHEABLE:
451 return ("UNCACHEABLE");
457 static int g_ntb_msix_idx = 1;
458 SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx,
459 0, "Use this memory window to access the peer MSIX message complex on "
460 "certain Xeon-based NTB systems, as a workaround for a hardware errata. "
461 "Like b2b_mw_idx, negative values index from the last available memory "
462 "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)");
464 static int g_ntb_mw_idx = -1;
465 SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx,
466 0, "Use this memory window to access the peer NTB registers. A "
467 "non-negative value starts from the first MW index; a negative value "
468 "starts from the last MW index. The default is -1, i.e., the last "
469 "available memory window. Both sides of the NTB MUST set the same "
470 "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)");
472 /* Hardware owns the low 16 bits of features. */
473 #define NTB_BAR_SIZE_4K (1 << 0)
474 #define NTB_SDOORBELL_LOCKUP (1 << 1)
475 #define NTB_SB01BASE_LOCKUP (1 << 2)
476 #define NTB_B2BDOORBELL_BIT14 (1 << 3)
477 /* Software/configuration owns the top 16 bits. */
478 #define NTB_SPLIT_BAR (1ull << 16)
480 #define NTB_FEATURES_STR \
481 "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \
482 "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K"
484 static struct ntb_hw_info pci_ids[] = {
485 /* XXX: PS/SS IDs left out until they are supported. */
486 { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B",
489 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
490 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
491 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
492 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
493 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
494 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
495 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K },
496 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,
497 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
498 NTB_SB01BASE_LOCKUP },
499 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,
500 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
501 NTB_SB01BASE_LOCKUP },
504 static const struct ntb_reg atom_reg = {
505 .ntb_ctl = ATOM_NTBCNTL_OFFSET,
506 .lnk_sta = ATOM_LINK_STATUS_OFFSET,
507 .db_size = sizeof(uint64_t),
508 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
511 static const struct ntb_alt_reg atom_pri_reg = {
512 .db_bell = ATOM_PDOORBELL_OFFSET,
513 .db_mask = ATOM_PDBMSK_OFFSET,
514 .spad = ATOM_SPAD_OFFSET,
517 static const struct ntb_alt_reg atom_b2b_reg = {
518 .db_bell = ATOM_B2B_DOORBELL_OFFSET,
519 .spad = ATOM_B2B_SPAD_OFFSET,
522 static const struct ntb_xlat_reg atom_sec_xlat = {
524 /* "FIXME" says the Linux driver. */
525 .bar0_base = ATOM_SBAR0BASE_OFFSET,
526 .bar2_base = ATOM_SBAR2BASE_OFFSET,
527 .bar4_base = ATOM_SBAR4BASE_OFFSET,
529 .bar2_limit = ATOM_SBAR2LMT_OFFSET,
530 .bar4_limit = ATOM_SBAR4LMT_OFFSET,
533 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET,
534 .bar4_xlat = ATOM_SBAR4XLAT_OFFSET,
537 static const struct ntb_reg xeon_reg = {
538 .ntb_ctl = XEON_NTBCNTL_OFFSET,
539 .lnk_sta = XEON_LINK_STATUS_OFFSET,
540 .db_size = sizeof(uint16_t),
541 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 },
544 static const struct ntb_alt_reg xeon_pri_reg = {
545 .db_bell = XEON_PDOORBELL_OFFSET,
546 .db_mask = XEON_PDBMSK_OFFSET,
547 .spad = XEON_SPAD_OFFSET,
550 static const struct ntb_alt_reg xeon_b2b_reg = {
551 .db_bell = XEON_B2B_DOORBELL_OFFSET,
552 .spad = XEON_B2B_SPAD_OFFSET,
555 static const struct ntb_xlat_reg xeon_sec_xlat = {
556 .bar0_base = XEON_SBAR0BASE_OFFSET,
557 .bar2_base = XEON_SBAR2BASE_OFFSET,
558 .bar4_base = XEON_SBAR4BASE_OFFSET,
559 .bar5_base = XEON_SBAR5BASE_OFFSET,
561 .bar2_limit = XEON_SBAR2LMT_OFFSET,
562 .bar4_limit = XEON_SBAR4LMT_OFFSET,
563 .bar5_limit = XEON_SBAR5LMT_OFFSET,
565 .bar2_xlat = XEON_SBAR2XLAT_OFFSET,
566 .bar4_xlat = XEON_SBAR4XLAT_OFFSET,
567 .bar5_xlat = XEON_SBAR5XLAT_OFFSET,
570 static struct ntb_b2b_addr xeon_b2b_usd_addr = {
571 .bar0_addr = XEON_B2B_BAR0_ADDR,
572 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
573 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
574 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
575 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
578 static struct ntb_b2b_addr xeon_b2b_dsd_addr = {
579 .bar0_addr = XEON_B2B_BAR0_ADDR,
580 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
581 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
582 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
583 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
586 SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0,
587 "B2B MW segment overrides -- MUST be the same on both sides");
589 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN,
590 &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "
591 "hardware, use this 64-bit address on the bus between the NTB devices for "
592 "the window at BAR2, on the upstream side of the link. MUST be the same "
593 "address on both sides.");
594 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN,
595 &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4.");
596 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN,
597 &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 "
598 "(split-BAR mode).");
599 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN,
600 &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 "
601 "(split-BAR mode).");
603 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN,
604 &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon "
605 "hardware, use this 64-bit address on the bus between the NTB devices for "
606 "the window at BAR2, on the downstream side of the link. MUST be the same"
607 " address on both sides.");
608 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN,
609 &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4.");
610 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN,
611 &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 "
612 "(split-BAR mode).");
613 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN,
614 &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 "
615 "(split-BAR mode).");
618 * OS <-> Driver interface structures
620 MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations");
623 * OS <-> Driver linkage functions
626 intel_ntb_probe(device_t device)
628 struct ntb_hw_info *p;
630 p = intel_ntb_get_device_info(pci_get_devid(device));
634 device_set_desc(device, p->desc);
639 intel_ntb_attach(device_t device)
641 struct ntb_softc *ntb;
642 struct ntb_hw_info *p;
645 ntb = device_get_softc(device);
646 p = intel_ntb_get_device_info(pci_get_devid(device));
648 ntb->device = device;
650 ntb->features = p->features;
651 ntb->b2b_mw_idx = B2B_MW_DISABLED;
652 ntb->msix_mw_idx = B2B_MW_DISABLED;
654 /* Heartbeat timer for NTB_ATOM since there is no link interrupt */
655 callout_init(&ntb->heartbeat_timer, 1);
656 callout_init(&ntb->lr_timer, 1);
657 callout_init(&ntb->peer_msix_work, 1);
658 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN);
660 if (ntb->type == NTB_ATOM)
661 error = intel_ntb_detect_atom(ntb);
663 error = intel_ntb_detect_xeon(ntb);
667 intel_ntb_detect_max_mw(ntb);
669 pci_enable_busmaster(ntb->device);
671 error = intel_ntb_map_pci_bars(ntb);
674 if (ntb->type == NTB_ATOM)
675 error = intel_ntb_atom_init_dev(ntb);
677 error = intel_ntb_xeon_init_dev(ntb);
681 intel_ntb_spad_clear(device);
683 intel_ntb_poll_link(ntb);
685 intel_ntb_sysctl_init(ntb);
687 /* Attach children to this controller */
688 error = ntb_register_device(device);
692 intel_ntb_detach(device);
697 intel_ntb_detach(device_t device)
699 struct ntb_softc *ntb;
701 ntb = device_get_softc(device);
703 /* Detach & delete all children */
704 ntb_unregister_device(device);
706 if (ntb->self_reg != NULL) {
708 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask);
711 callout_drain(&ntb->heartbeat_timer);
712 callout_drain(&ntb->lr_timer);
713 callout_drain(&ntb->peer_msix_work);
714 pci_disable_busmaster(ntb->device);
715 if (ntb->type == NTB_XEON)
716 intel_ntb_teardown_xeon(ntb);
717 intel_ntb_teardown_interrupts(ntb);
719 mtx_destroy(&ntb->db_mask_lock);
721 intel_ntb_unmap_pci_bar(ntb);
727 * Driver internal routines
729 static inline enum ntb_bar
730 intel_ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw)
733 KASSERT(mw < ntb->mw_count,
734 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count));
735 KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw"));
737 return (ntb->reg->mw_bar[mw]);
741 bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar)
743 /* XXX This assertion could be stronger. */
744 KASSERT(bar < NTB_MAX_BARS, ("bogus bar"));
745 return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR));
749 bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base,
750 uint32_t *xlat, uint32_t *lmt)
752 uint32_t basev, lmtv, xlatv;
756 basev = ntb->xlat_reg->bar2_base;
757 lmtv = ntb->xlat_reg->bar2_limit;
758 xlatv = ntb->xlat_reg->bar2_xlat;
761 basev = ntb->xlat_reg->bar4_base;
762 lmtv = ntb->xlat_reg->bar4_limit;
763 xlatv = ntb->xlat_reg->bar4_xlat;
766 basev = ntb->xlat_reg->bar5_base;
767 lmtv = ntb->xlat_reg->bar5_limit;
768 xlatv = ntb->xlat_reg->bar5_xlat;
771 KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS,
773 basev = lmtv = xlatv = 0;
786 intel_ntb_map_pci_bars(struct ntb_softc *ntb)
788 struct ntb_pci_bar_info *bar;
791 bar = &ntb->bar_info[NTB_CONFIG_BAR];
792 bar->pci_resource_id = PCIR_BAR(0);
793 rc = map_mmr_bar(ntb, bar);
798 * At least on Xeon v4 NTB device leaks to host some remote side
799 * BAR0 writes supposed to update scratchpad registers. I am not
800 * sure why it happens, but it may be related to the fact that
801 * on a link side BAR0 is 32KB, while on a host side it is 64KB.
802 * Without this hack DMAR blocks those accesses as not allowed.
804 if (bus_dma_tag_create(bus_get_dma_tag(ntb->device), 1, 0,
805 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
806 bar->size, 1, bar->size, 0, NULL, NULL, &ntb->bar0_dma_tag)) {
807 device_printf(ntb->device, "Unable to create BAR0 tag\n");
810 if (bus_dmamap_create(ntb->bar0_dma_tag, 0, &ntb->bar0_dma_map)) {
811 device_printf(ntb->device, "Unable to create BAR0 map\n");
814 if (bus_dma_dmar_load_ident(ntb->bar0_dma_tag, ntb->bar0_dma_map,
815 bar->pbase, bar->size, 0)) {
816 device_printf(ntb->device, "Unable to load BAR0 map\n");
820 bar = &ntb->bar_info[NTB_B2B_BAR_1];
821 bar->pci_resource_id = PCIR_BAR(2);
822 rc = map_memory_window_bar(ntb, bar);
825 bar->psz_off = XEON_PBAR23SZ_OFFSET;
826 bar->ssz_off = XEON_SBAR23SZ_OFFSET;
827 bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
829 bar = &ntb->bar_info[NTB_B2B_BAR_2];
830 bar->pci_resource_id = PCIR_BAR(4);
831 rc = map_memory_window_bar(ntb, bar);
834 bar->psz_off = XEON_PBAR4SZ_OFFSET;
835 bar->ssz_off = XEON_SBAR4SZ_OFFSET;
836 bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
838 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR))
841 bar = &ntb->bar_info[NTB_B2B_BAR_3];
842 bar->pci_resource_id = PCIR_BAR(5);
843 rc = map_memory_window_bar(ntb, bar);
844 bar->psz_off = XEON_PBAR5SZ_OFFSET;
845 bar->ssz_off = XEON_SBAR5SZ_OFFSET;
846 bar->pbarxlat_off = XEON_PBAR5XLAT_OFFSET;
850 device_printf(ntb->device,
851 "unable to allocate pci resource\n");
856 print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar,
860 device_printf(ntb->device,
861 "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n",
862 PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
863 (char *)bar->vbase + bar->size - 1,
864 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1),
865 (uintmax_t)bar->size, kind);
869 map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
872 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
873 &bar->pci_resource_id, RF_ACTIVE);
874 if (bar->pci_resource == NULL)
877 save_bar_parameters(bar);
878 bar->map_mode = VM_MEMATTR_UNCACHEABLE;
879 print_map_success(ntb, bar, "mmr");
884 map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar)
887 vm_memattr_t mapmode;
888 uint8_t bar_size_bits = 0;
890 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY,
891 &bar->pci_resource_id, RF_ACTIVE);
893 if (bar->pci_resource == NULL)
896 save_bar_parameters(bar);
898 * Ivytown NTB BAR sizes are misreported by the hardware due to a
899 * hardware issue. To work around this, query the size it should be
900 * configured to by the device and modify the resource to correspond to
901 * this new size. The BIOS on systems with this problem is required to
902 * provide enough address space to allow the driver to make this change
905 * Ideally I could have just specified the size when I allocated the
907 * bus_alloc_resource(ntb->device,
908 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul,
909 * 1ul << bar_size_bits, RF_ACTIVE);
910 * but the PCI driver does not honor the size in this call, so we have
911 * to modify it after the fact.
913 if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) {
914 if (bar->pci_resource_id == PCIR_BAR(2))
915 bar_size_bits = pci_read_config(ntb->device,
916 XEON_PBAR23SZ_OFFSET, 1);
918 bar_size_bits = pci_read_config(ntb->device,
919 XEON_PBAR45SZ_OFFSET, 1);
921 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY,
922 bar->pci_resource, bar->pbase,
923 bar->pbase + (1ul << bar_size_bits) - 1);
925 device_printf(ntb->device,
926 "unable to resize bar\n");
930 save_bar_parameters(bar);
933 bar->map_mode = VM_MEMATTR_UNCACHEABLE;
934 print_map_success(ntb, bar, "mw");
937 * Optionally, mark MW BARs as anything other than UC to improve
940 mapmode = intel_ntb_pat_flags();
941 if (mapmode == bar->map_mode)
944 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode);
946 bar->map_mode = mapmode;
947 device_printf(ntb->device,
948 "Marked BAR%d v:[%p-%p] p:[%p-%p] as "
950 PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
951 (char *)bar->vbase + bar->size - 1,
952 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1),
953 intel_ntb_vm_memattr_to_str(mapmode));
955 device_printf(ntb->device,
956 "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as "
958 PCI_RID2BAR(bar->pci_resource_id), bar->vbase,
959 (char *)bar->vbase + bar->size - 1,
960 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1),
961 intel_ntb_vm_memattr_to_str(mapmode), rc);
967 intel_ntb_unmap_pci_bar(struct ntb_softc *ntb)
969 struct ntb_pci_bar_info *bar;
972 if (ntb->bar0_dma_map != NULL) {
973 bus_dmamap_unload(ntb->bar0_dma_tag, ntb->bar0_dma_map);
974 bus_dmamap_destroy(ntb->bar0_dma_tag, ntb->bar0_dma_map);
976 if (ntb->bar0_dma_tag != NULL)
977 bus_dma_tag_destroy(ntb->bar0_dma_tag);
978 for (i = 0; i < NTB_MAX_BARS; i++) {
979 bar = &ntb->bar_info[i];
980 if (bar->pci_resource != NULL)
981 bus_release_resource(ntb->device, SYS_RES_MEMORY,
982 bar->pci_resource_id, bar->pci_resource);
987 intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors)
992 for (i = 0; i < num_vectors; i++) {
993 ntb->int_info[i].rid = i + 1;
994 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device,
995 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE);
996 if (ntb->int_info[i].res == NULL) {
997 device_printf(ntb->device,
998 "bus_alloc_resource failed\n");
1001 ntb->int_info[i].tag = NULL;
1002 ntb->allocated_interrupts++;
1003 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res,
1004 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr,
1005 &ntb->msix_vec[i], &ntb->int_info[i].tag);
1007 device_printf(ntb->device, "bus_setup_intr failed\n");
1015 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector
1016 * cannot be allocated for each MSI-X message. JHB seems to think remapping
1017 * should be okay. This tunable should enable us to test that hypothesis
1018 * when someone gets their hands on some Xeon hardware.
1020 static int ntb_force_remap_mode;
1021 SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN,
1022 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped"
1023 " to a smaller number of ithreads, even if the desired number are "
1027 * In case it is NOT ok, give consumers an abort button.
1029 static int ntb_prefer_intx;
1030 SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN,
1031 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather "
1032 "than remapping MSI-X messages over available slots (match Linux driver "
1036 * Remap the desired number of MSI-X messages to available ithreads in a simple
1037 * round-robin fashion.
1040 intel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail)
1046 if (ntb_prefer_intx != 0)
1049 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK);
1051 for (i = 0; i < desired; i++)
1052 vectors[i] = (i % avail) + 1;
1054 rc = pci_remap_msix(dev, desired, vectors);
1055 free(vectors, M_NTB);
1060 intel_ntb_init_isr(struct ntb_softc *ntb)
1062 uint32_t desired_vectors, num_vectors;
1065 ntb->allocated_interrupts = 0;
1066 ntb->last_ts = ticks;
1069 * Mask all doorbell interrupts. (Except link events!)
1072 ntb->db_mask = ntb->db_valid_mask;
1073 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
1074 DB_MASK_UNLOCK(ntb);
1076 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
1078 if (desired_vectors >= 1) {
1079 rc = pci_alloc_msix(ntb->device, &num_vectors);
1081 if (ntb_force_remap_mode != 0 && rc == 0 &&
1082 num_vectors == desired_vectors)
1085 if (rc == 0 && num_vectors < desired_vectors) {
1086 rc = intel_ntb_remap_msix(ntb->device, desired_vectors,
1089 num_vectors = desired_vectors;
1091 pci_release_msi(ntb->device);
1098 if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
1099 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
1100 device_printf(ntb->device,
1101 "Errata workaround does not support MSI or INTX\n");
1105 ntb->db_vec_count = 1;
1106 ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT;
1107 rc = intel_ntb_setup_legacy_interrupt(ntb);
1109 if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS &&
1110 HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
1111 device_printf(ntb->device,
1112 "Errata workaround expects %d doorbell bits\n",
1113 XEON_NONLINK_DB_MSIX_BITS);
1117 intel_ntb_create_msix_vec(ntb, num_vectors);
1118 rc = intel_ntb_setup_msix(ntb, num_vectors);
1121 device_printf(ntb->device,
1122 "Error allocating interrupts: %d\n", rc);
1123 intel_ntb_free_msix_vec(ntb);
1130 intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb)
1134 ntb->int_info[0].rid = 0;
1135 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ,
1136 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE);
1137 if (ntb->int_info[0].res == NULL) {
1138 device_printf(ntb->device, "bus_alloc_resource failed\n");
1142 ntb->int_info[0].tag = NULL;
1143 ntb->allocated_interrupts = 1;
1145 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res,
1146 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr,
1147 ntb, &ntb->int_info[0].tag);
1149 device_printf(ntb->device, "bus_setup_intr failed\n");
1157 intel_ntb_teardown_interrupts(struct ntb_softc *ntb)
1159 struct ntb_int_info *current_int;
1162 for (i = 0; i < ntb->allocated_interrupts; i++) {
1163 current_int = &ntb->int_info[i];
1164 if (current_int->tag != NULL)
1165 bus_teardown_intr(ntb->device, current_int->res,
1168 if (current_int->res != NULL)
1169 bus_release_resource(ntb->device, SYS_RES_IRQ,
1170 rman_get_rid(current_int->res), current_int->res);
1173 intel_ntb_free_msix_vec(ntb);
1174 pci_release_msi(ntb->device);
1178 * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it
1179 * out to make code clearer.
1181 static inline uint64_t
1182 db_ioread(struct ntb_softc *ntb, uint64_t regoff)
1185 if (ntb->type == NTB_ATOM)
1186 return (intel_ntb_reg_read(8, regoff));
1188 KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
1190 return (intel_ntb_reg_read(2, regoff));
1194 db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
1197 KASSERT((val & ~ntb->db_valid_mask) == 0,
1198 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
1199 (uintmax_t)(val & ~ntb->db_valid_mask),
1200 (uintmax_t)ntb->db_valid_mask));
1202 if (regoff == ntb->self_reg->db_mask)
1203 DB_MASK_ASSERT(ntb, MA_OWNED);
1204 db_iowrite_raw(ntb, regoff, val);
1208 db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
1211 if (ntb->type == NTB_ATOM) {
1212 intel_ntb_reg_write(8, regoff, val);
1216 KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
1217 intel_ntb_reg_write(2, regoff, (uint16_t)val);
1221 intel_ntb_db_set_mask(device_t dev, uint64_t bits)
1223 struct ntb_softc *ntb = device_get_softc(dev);
1226 ntb->db_mask |= bits;
1227 if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
1228 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
1229 DB_MASK_UNLOCK(ntb);
1233 intel_ntb_db_clear_mask(device_t dev, uint64_t bits)
1235 struct ntb_softc *ntb = device_get_softc(dev);
1239 KASSERT((bits & ~ntb->db_valid_mask) == 0,
1240 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
1241 (uintmax_t)(bits & ~ntb->db_valid_mask),
1242 (uintmax_t)ntb->db_valid_mask));
1245 ibits = ntb->fake_db & ntb->db_mask & bits;
1246 ntb->db_mask &= ~bits;
1247 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
1248 /* Simulate fake interrupts if unmasked DB bits are set. */
1249 ntb->force_db |= ibits;
1250 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
1251 if ((ibits & intel_ntb_db_vector_mask(dev, i)) != 0)
1252 swi_sched(ntb->int_info[i].tag, 0);
1255 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
1257 DB_MASK_UNLOCK(ntb);
1261 intel_ntb_db_read(device_t dev)
1263 struct ntb_softc *ntb = device_get_softc(dev);
1265 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
1266 return (ntb->fake_db);
1268 return (db_ioread(ntb, ntb->self_reg->db_bell));
1272 intel_ntb_db_clear(device_t dev, uint64_t bits)
1274 struct ntb_softc *ntb = device_get_softc(dev);
1276 KASSERT((bits & ~ntb->db_valid_mask) == 0,
1277 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__,
1278 (uintmax_t)(bits & ~ntb->db_valid_mask),
1279 (uintmax_t)ntb->db_valid_mask));
1281 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
1283 ntb->fake_db &= ~bits;
1284 DB_MASK_UNLOCK(ntb);
1288 db_iowrite(ntb, ntb->self_reg->db_bell, bits);
1291 static inline uint64_t
1292 intel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector)
1294 uint64_t shift, mask;
1296 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
1298 * Remap vectors in custom way to make at least first
1299 * three doorbells to not generate stray events.
1300 * This breaks Linux compatibility (if one existed)
1301 * when more then one DB is used (not by if_ntb).
1303 if (db_vector < XEON_NONLINK_DB_MSIX_BITS - 1)
1304 return (1 << db_vector);
1305 if (db_vector == XEON_NONLINK_DB_MSIX_BITS - 1)
1309 shift = ntb->db_vec_shift;
1310 mask = (1ull << shift) - 1;
1311 return (mask << (shift * db_vector));
1315 intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
1319 ntb->last_ts = ticks;
1320 vec_mask = intel_ntb_vec_mask(ntb, vec);
1322 if ((vec_mask & ntb->db_link_mask) != 0) {
1323 if (intel_ntb_poll_link(ntb))
1324 ntb_link_event(ntb->device);
1327 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&
1328 (vec_mask & ntb->db_link_mask) == 0) {
1332 * Do not report same DB events again if not cleared yet,
1333 * unless the mask was just cleared for them and this
1334 * interrupt handler call can be the consequence of it.
1336 vec_mask &= ~ntb->fake_db | ntb->force_db;
1337 ntb->force_db &= ~vec_mask;
1339 /* Update our internal doorbell register. */
1340 ntb->fake_db |= vec_mask;
1342 /* Do not report masked DB events. */
1343 vec_mask &= ~ntb->db_mask;
1345 DB_MASK_UNLOCK(ntb);
1348 if ((vec_mask & ntb->db_valid_mask) != 0)
1349 ntb_db_event(ntb->device, vec);
1353 ndev_vec_isr(void *arg)
1355 struct ntb_vec *nvec = arg;
1357 intel_ntb_interrupt(nvec->ntb, nvec->num);
1361 ndev_irq_isr(void *arg)
1363 /* If we couldn't set up MSI-X, we only have the one vector. */
1364 intel_ntb_interrupt(arg, 0);
1368 intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors)
1372 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB,
1374 for (i = 0; i < num_vectors; i++) {
1375 ntb->msix_vec[i].num = i;
1376 ntb->msix_vec[i].ntb = ntb;
1383 intel_ntb_free_msix_vec(struct ntb_softc *ntb)
1386 if (ntb->msix_vec == NULL)
1389 free(ntb->msix_vec, M_NTB);
1390 ntb->msix_vec = NULL;
1394 intel_ntb_get_msix_info(struct ntb_softc *ntb)
1396 struct pci_devinfo *dinfo;
1397 struct pcicfg_msix *msix;
1398 uint32_t laddr, data, i, offset;
1400 dinfo = device_get_ivars(ntb->device);
1401 msix = &dinfo->cfg.msix;
1403 CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data));
1405 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
1406 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE;
1408 laddr = bus_read_4(msix->msix_table_res, offset +
1409 PCI_MSIX_ENTRY_LOWER_ADDR);
1410 intel_ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr);
1412 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE,
1413 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr,
1414 MSI_INTEL_ADDR_BASE));
1415 ntb->msix_data[i].nmd_ofs = laddr;
1417 data = bus_read_4(msix->msix_table_res, offset +
1418 PCI_MSIX_ENTRY_DATA);
1419 intel_ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data);
1421 ntb->msix_data[i].nmd_data = data;
1425 static struct ntb_hw_info *
1426 intel_ntb_get_device_info(uint32_t device_id)
1428 struct ntb_hw_info *ep;
1430 for (ep = pci_ids; ep < &pci_ids[nitems(pci_ids)]; ep++) {
1431 if (ep->device_id == device_id)
1438 intel_ntb_teardown_xeon(struct ntb_softc *ntb)
1441 if (ntb->reg != NULL)
1442 intel_ntb_link_disable(ntb->device);
1446 intel_ntb_detect_max_mw(struct ntb_softc *ntb)
1449 if (ntb->type == NTB_ATOM) {
1450 ntb->mw_count = ATOM_MW_COUNT;
1454 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
1455 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
1457 ntb->mw_count = XEON_SNB_MW_COUNT;
1461 intel_ntb_detect_xeon(struct ntb_softc *ntb)
1463 uint8_t ppd, conn_type;
1465 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1);
1468 if ((ppd & XEON_PPD_DEV_TYPE) != 0)
1469 ntb->dev_type = NTB_DEV_DSD;
1471 ntb->dev_type = NTB_DEV_USD;
1473 if ((ppd & XEON_PPD_SPLIT_BAR) != 0)
1474 ntb->features |= NTB_SPLIT_BAR;
1476 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&
1477 !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
1478 device_printf(ntb->device,
1479 "Can not apply SB01BASE_LOCKUP workaround "
1480 "with split BARs disabled!\n");
1481 device_printf(ntb->device,
1482 "Expect system hangs under heavy NTB traffic!\n");
1483 ntb->features &= ~NTB_SB01BASE_LOCKUP;
1487 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP
1488 * errata workaround; only do one at a time.
1490 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
1491 ntb->features &= ~NTB_SDOORBELL_LOCKUP;
1493 conn_type = ppd & XEON_PPD_CONN_TYPE;
1494 switch (conn_type) {
1496 ntb->conn_type = conn_type;
1499 case NTB_CONN_TRANSPARENT:
1501 device_printf(ntb->device, "Unsupported connection type: %u\n",
1502 (unsigned)conn_type);
1509 intel_ntb_detect_atom(struct ntb_softc *ntb)
1511 uint32_t ppd, conn_type;
1513 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
1516 if ((ppd & ATOM_PPD_DEV_TYPE) != 0)
1517 ntb->dev_type = NTB_DEV_DSD;
1519 ntb->dev_type = NTB_DEV_USD;
1521 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8;
1522 switch (conn_type) {
1524 ntb->conn_type = conn_type;
1527 device_printf(ntb->device, "Unsupported NTB configuration\n");
1534 intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
1538 ntb->spad_count = XEON_SPAD_COUNT;
1539 ntb->db_count = XEON_DB_COUNT;
1540 ntb->db_link_mask = XEON_DB_LINK_BIT;
1541 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT;
1542 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT;
1544 if (ntb->conn_type != NTB_CONN_B2B) {
1545 device_printf(ntb->device, "Connection type %d not supported\n",
1550 ntb->reg = &xeon_reg;
1551 ntb->self_reg = &xeon_pri_reg;
1552 ntb->peer_reg = &xeon_b2b_reg;
1553 ntb->xlat_reg = &xeon_sec_xlat;
1555 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
1556 ntb->force_db = ntb->fake_db = 0;
1557 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) %
1559 intel_ntb_printf(2, "Setting up MSIX mw idx %d means %u\n",
1560 g_ntb_msix_idx, ntb->msix_mw_idx);
1561 rc = intel_ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx,
1562 VM_MEMATTR_UNCACHEABLE);
1563 KASSERT(rc == 0, ("shouldn't fail"));
1564 } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
1566 * There is a Xeon hardware errata related to writes to SDOORBELL or
1567 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space,
1568 * which may hang the system. To workaround this, use a memory
1569 * window to access the interrupt and scratch pad registers on the
1572 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) %
1574 intel_ntb_printf(2, "Setting up b2b mw idx %d means %u\n",
1575 g_ntb_mw_idx, ntb->b2b_mw_idx);
1576 rc = intel_ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx,
1577 VM_MEMATTR_UNCACHEABLE);
1578 KASSERT(rc == 0, ("shouldn't fail"));
1579 } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14))
1581 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be
1582 * mirrored to the remote system. Shrink the number of bits by one,
1583 * since bit 14 is the last bit.
1585 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register
1586 * anyway. Nor for non-B2B connection types.
1588 ntb->db_count = XEON_DB_COUNT - 1;
1590 ntb->db_valid_mask = (1ull << ntb->db_count) - 1;
1592 if (ntb->dev_type == NTB_DEV_USD)
1593 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr,
1594 &xeon_b2b_usd_addr);
1596 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr,
1597 &xeon_b2b_dsd_addr);
1601 /* Enable Bus Master and Memory Space on the secondary side */
1602 intel_ntb_reg_write(2, XEON_SPCICMD_OFFSET,
1603 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
1606 * Mask all doorbell interrupts.
1609 ntb->db_mask = ntb->db_valid_mask;
1610 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask);
1611 DB_MASK_UNLOCK(ntb);
1613 rc = intel_ntb_init_isr(ntb);
1618 intel_ntb_atom_init_dev(struct ntb_softc *ntb)
1622 KASSERT(ntb->conn_type == NTB_CONN_B2B,
1623 ("Unsupported NTB configuration (%d)\n", ntb->conn_type));
1625 ntb->spad_count = ATOM_SPAD_COUNT;
1626 ntb->db_count = ATOM_DB_COUNT;
1627 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT;
1628 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT;
1629 ntb->db_valid_mask = (1ull << ntb->db_count) - 1;
1631 ntb->reg = &atom_reg;
1632 ntb->self_reg = &atom_pri_reg;
1633 ntb->peer_reg = &atom_b2b_reg;
1634 ntb->xlat_reg = &atom_sec_xlat;
1637 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is
1638 * resolved. Mask transaction layer internal parity errors.
1640 pci_write_config(ntb->device, 0xFC, 0x4, 4);
1642 configure_atom_secondary_side_bars(ntb);
1644 /* Enable Bus Master and Memory Space on the secondary side */
1645 intel_ntb_reg_write(2, ATOM_SPCICMD_OFFSET,
1646 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
1648 error = intel_ntb_init_isr(ntb);
1652 /* Initiate PCI-E link training */
1653 intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
1655 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb);
1660 /* XXX: Linux driver doesn't seem to do any of this for Atom. */
1662 configure_atom_secondary_side_bars(struct ntb_softc *ntb)
1665 if (ntb->dev_type == NTB_DEV_USD) {
1666 intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET,
1667 XEON_B2B_BAR2_ADDR64);
1668 intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET,
1669 XEON_B2B_BAR4_ADDR64);
1670 intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64);
1671 intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64);
1673 intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET,
1674 XEON_B2B_BAR2_ADDR64);
1675 intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET,
1676 XEON_B2B_BAR4_ADDR64);
1677 intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64);
1678 intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64);
1684 * When working around Xeon SDOORBELL errata by remapping remote registers in a
1685 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW
1686 * remains for use by a higher layer.
1688 * Will only be used if working around SDOORBELL errata and the BIOS-configured
1689 * MW size is sufficiently large.
1691 static unsigned int ntb_b2b_mw_share;
1692 SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share,
1693 0, "If enabled (non-zero), prefer to share half of the B2B peer register "
1694 "MW with higher level consumers. Both sides of the NTB MUST set the same "
1698 xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx,
1699 enum ntb_bar regbar)
1701 struct ntb_pci_bar_info *bar;
1704 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3)
1707 bar = &ntb->bar_info[idx];
1708 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1);
1709 if (idx == regbar) {
1710 if (ntb->b2b_off != 0)
1715 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1);
1716 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1);
1721 xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr,
1722 enum ntb_bar idx, enum ntb_bar regbar)
1725 uint32_t base_reg, lmt_reg;
1727 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg);
1728 if (idx == regbar) {
1730 bar_addr += ntb->b2b_off;
1735 if (!bar_is_64bit(ntb, idx)) {
1736 intel_ntb_reg_write(4, base_reg, bar_addr);
1737 reg_val = intel_ntb_reg_read(4, base_reg);
1740 intel_ntb_reg_write(4, lmt_reg, bar_addr);
1741 reg_val = intel_ntb_reg_read(4, lmt_reg);
1744 intel_ntb_reg_write(8, base_reg, bar_addr);
1745 reg_val = intel_ntb_reg_read(8, base_reg);
1748 intel_ntb_reg_write(8, lmt_reg, bar_addr);
1749 reg_val = intel_ntb_reg_read(8, lmt_reg);
1755 xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx)
1757 struct ntb_pci_bar_info *bar;
1759 bar = &ntb->bar_info[idx];
1760 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) {
1761 intel_ntb_reg_write(4, bar->pbarxlat_off, base_addr);
1762 base_addr = intel_ntb_reg_read(4, bar->pbarxlat_off);
1764 intel_ntb_reg_write(8, bar->pbarxlat_off, base_addr);
1765 base_addr = intel_ntb_reg_read(8, bar->pbarxlat_off);
1771 xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr,
1772 const struct ntb_b2b_addr *peer_addr)
1774 struct ntb_pci_bar_info *b2b_bar;
1777 enum ntb_bar b2b_bar_num, i;
1779 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) {
1781 b2b_bar_num = NTB_CONFIG_BAR;
1784 b2b_bar_num = intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx);
1785 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS,
1786 ("invalid b2b mw bar"));
1788 b2b_bar = &ntb->bar_info[b2b_bar_num];
1789 bar_size = b2b_bar->size;
1791 if (ntb_b2b_mw_share != 0 &&
1792 (bar_size >> 1) >= XEON_B2B_MIN_SIZE)
1793 ntb->b2b_off = bar_size >> 1;
1794 else if (bar_size >= XEON_B2B_MIN_SIZE) {
1797 device_printf(ntb->device,
1798 "B2B bar size is too small!\n");
1804 * Reset the secondary bar sizes to match the primary bar sizes.
1805 * (Except, disable or halve the size of the B2B secondary bar.)
1807 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++)
1808 xeon_reset_sbar_size(ntb, i, b2b_bar_num);
1811 if (b2b_bar_num == NTB_CONFIG_BAR)
1812 bar_addr = addr->bar0_addr;
1813 else if (b2b_bar_num == NTB_B2B_BAR_1)
1814 bar_addr = addr->bar2_addr64;
1815 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR))
1816 bar_addr = addr->bar4_addr64;
1817 else if (b2b_bar_num == NTB_B2B_BAR_2)
1818 bar_addr = addr->bar4_addr32;
1819 else if (b2b_bar_num == NTB_B2B_BAR_3)
1820 bar_addr = addr->bar5_addr32;
1822 KASSERT(false, ("invalid bar"));
1824 intel_ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr);
1827 * Other SBARs are normally hit by the PBAR xlat, except for the b2b
1828 * register BAR. The B2B BAR is either disabled above or configured
1829 * half-size. It starts at PBAR xlat + offset.
1831 * Also set up incoming BAR limits == base (zero length window).
1833 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1,
1835 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
1836 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32,
1837 NTB_B2B_BAR_2, b2b_bar_num);
1838 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32,
1839 NTB_B2B_BAR_3, b2b_bar_num);
1841 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64,
1842 NTB_B2B_BAR_2, b2b_bar_num);
1844 /* Zero incoming translation addrs */
1845 intel_ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0);
1846 intel_ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0);
1848 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
1849 uint32_t xlat_reg, lmt_reg;
1850 enum ntb_bar bar_num;
1853 * We point the chosen MSIX MW BAR xlat to remote LAPIC for
1856 bar_num = intel_ntb_mw_to_bar(ntb, ntb->msix_mw_idx);
1857 bar_get_xlat_params(ntb, bar_num, NULL, &xlat_reg, &lmt_reg);
1858 if (bar_is_64bit(ntb, bar_num)) {
1859 intel_ntb_reg_write(8, xlat_reg, MSI_INTEL_ADDR_BASE);
1860 ntb->msix_xlat = intel_ntb_reg_read(8, xlat_reg);
1861 intel_ntb_reg_write(8, lmt_reg, 0);
1863 intel_ntb_reg_write(4, xlat_reg, MSI_INTEL_ADDR_BASE);
1864 ntb->msix_xlat = intel_ntb_reg_read(4, xlat_reg);
1865 intel_ntb_reg_write(4, lmt_reg, 0);
1868 ntb->peer_lapic_bar = &ntb->bar_info[bar_num];
1870 (void)intel_ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET);
1871 (void)intel_ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET);
1873 /* Zero outgoing translation limits (whole bar size windows) */
1874 intel_ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0);
1875 intel_ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0);
1877 /* Set outgoing translation offsets */
1878 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1);
1879 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
1880 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2);
1881 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3);
1883 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2);
1885 /* Set the translation offset for B2B registers */
1887 if (b2b_bar_num == NTB_CONFIG_BAR)
1888 bar_addr = peer_addr->bar0_addr;
1889 else if (b2b_bar_num == NTB_B2B_BAR_1)
1890 bar_addr = peer_addr->bar2_addr64;
1891 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR))
1892 bar_addr = peer_addr->bar4_addr64;
1893 else if (b2b_bar_num == NTB_B2B_BAR_2)
1894 bar_addr = peer_addr->bar4_addr32;
1895 else if (b2b_bar_num == NTB_B2B_BAR_3)
1896 bar_addr = peer_addr->bar5_addr32;
1898 KASSERT(false, ("invalid bar"));
1901 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits
1904 intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff);
1905 intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32);
1910 _xeon_link_is_up(struct ntb_softc *ntb)
1913 if (ntb->conn_type == NTB_CONN_TRANSPARENT)
1915 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0);
1919 link_is_up(struct ntb_softc *ntb)
1922 if (ntb->type == NTB_XEON)
1923 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good ||
1924 !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)));
1926 KASSERT(ntb->type == NTB_ATOM, ("ntb type"));
1927 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0);
1931 atom_link_is_err(struct ntb_softc *ntb)
1935 KASSERT(ntb->type == NTB_ATOM, ("ntb type"));
1937 status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET);
1938 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0)
1941 status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET);
1942 return ((status & ATOM_IBIST_ERR_OFLOW) != 0);
1945 /* Atom does not have link status interrupt, poll on that platform */
1947 atom_link_hb(void *arg)
1949 struct ntb_softc *ntb = arg;
1950 sbintime_t timo, poll_ts;
1952 timo = NTB_HB_TIMEOUT * hz;
1953 poll_ts = ntb->last_ts + timo;
1956 * Delay polling the link status if an interrupt was received, unless
1957 * the cached link status says the link is down.
1959 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) {
1960 timo = poll_ts - ticks;
1964 if (intel_ntb_poll_link(ntb))
1965 ntb_link_event(ntb->device);
1967 if (!link_is_up(ntb) && atom_link_is_err(ntb)) {
1968 /* Link is down with error, proceed with recovery */
1969 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb);
1974 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb);
1978 atom_perform_link_restart(struct ntb_softc *ntb)
1982 /* Driver resets the NTB ModPhy lanes - magic! */
1983 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0);
1984 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40);
1985 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60);
1986 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60);
1988 /* Driver waits 100ms to allow the NTB ModPhy to settle */
1989 pause("ModPhy", hz / 10);
1991 /* Clear AER Errors, write to clear */
1992 status = intel_ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET);
1993 status &= PCIM_AER_COR_REPLAY_ROLLOVER;
1994 intel_ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status);
1996 /* Clear unexpected electrical idle event in LTSSM, write to clear */
1997 status = intel_ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET);
1998 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1999 intel_ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status);
2001 /* Clear DeSkew Buffer error, write to clear */
2002 status = intel_ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET);
2003 status |= ATOM_DESKEWSTS_DBERR;
2004 intel_ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status);
2006 status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET);
2007 status &= ATOM_IBIST_ERR_OFLOW;
2008 intel_ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status);
2010 /* Releases the NTB state machine to allow the link to retrain */
2011 status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET);
2012 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
2013 intel_ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status);
2017 intel_ntb_port_number(device_t dev)
2019 struct ntb_softc *ntb = device_get_softc(dev);
2021 return (ntb->dev_type == NTB_DEV_USD ? 0 : 1);
2025 intel_ntb_peer_port_count(device_t dev)
2032 intel_ntb_peer_port_number(device_t dev, int pidx)
2034 struct ntb_softc *ntb = device_get_softc(dev);
2039 return (ntb->dev_type == NTB_DEV_USD ? 1 : 0);
2043 intel_ntb_peer_port_idx(device_t dev, int port)
2047 peer_port = intel_ntb_peer_port_number(dev, 0);
2048 if (peer_port == -EINVAL || port != peer_port)
2055 intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused,
2056 enum ntb_width width __unused)
2058 struct ntb_softc *ntb = device_get_softc(dev);
2061 intel_ntb_printf(2, "%s\n", __func__);
2063 if (ntb->type == NTB_ATOM) {
2064 pci_write_config(ntb->device, NTB_PPD_OFFSET,
2065 ntb->ppd | ATOM_PPD_INIT_LINK, 4);
2069 if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
2070 ntb_link_event(dev);
2074 cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
2075 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
2076 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
2077 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP;
2078 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
2079 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP;
2080 intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
2085 intel_ntb_link_disable(device_t dev)
2087 struct ntb_softc *ntb = device_get_softc(dev);
2090 intel_ntb_printf(2, "%s\n", __func__);
2092 if (ntb->conn_type == NTB_CONN_TRANSPARENT) {
2093 ntb_link_event(dev);
2097 cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
2098 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
2099 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
2100 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
2101 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP);
2102 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
2103 intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl);
2108 intel_ntb_link_enabled(device_t dev)
2110 struct ntb_softc *ntb = device_get_softc(dev);
2113 if (ntb->type == NTB_ATOM) {
2114 cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4);
2115 return ((cntl & ATOM_PPD_INIT_LINK) != 0);
2118 if (ntb->conn_type == NTB_CONN_TRANSPARENT)
2121 cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
2122 return ((cntl & NTB_CNTL_LINK_DISABLE) == 0);
2126 recover_atom_link(void *arg)
2128 struct ntb_softc *ntb = arg;
2129 unsigned speed, width, oldspeed, oldwidth;
2132 atom_perform_link_restart(ntb);
2135 * There is a potential race between the 2 NTB devices recovering at
2136 * the same time. If the times are the same, the link will not recover
2137 * and the driver will be stuck in this loop forever. Add a random
2138 * interval to the recovery time to prevent this race.
2140 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME;
2141 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000);
2143 if (atom_link_is_err(ntb))
2146 status32 = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
2147 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0)
2150 status32 = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
2151 width = NTB_LNK_STA_WIDTH(status32);
2152 speed = status32 & NTB_LINK_SPEED_MASK;
2154 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta);
2155 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK;
2156 if (oldwidth != width || oldspeed != speed)
2160 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb,
2165 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link,
2170 * Polls the HW link status register(s); returns true if something has changed.
2173 intel_ntb_poll_link(struct ntb_softc *ntb)
2178 if (ntb->type == NTB_ATOM) {
2179 ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl);
2180 if (ntb_cntl == ntb->ntb_ctl)
2183 ntb->ntb_ctl = ntb_cntl;
2184 ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
2186 db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask);
2188 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
2189 if (reg_val == ntb->lnk_sta)
2192 ntb->lnk_sta = reg_val;
2194 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
2195 if (_xeon_link_is_up(ntb)) {
2196 if (!ntb->peer_msix_good) {
2197 callout_reset(&ntb->peer_msix_work, 0,
2198 intel_ntb_exchange_msix, ntb);
2202 ntb->peer_msix_good = false;
2203 ntb->peer_msix_done = false;
2210 static inline enum ntb_speed
2211 intel_ntb_link_sta_speed(struct ntb_softc *ntb)
2214 if (!link_is_up(ntb))
2215 return (NTB_SPEED_NONE);
2216 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK);
2219 static inline enum ntb_width
2220 intel_ntb_link_sta_width(struct ntb_softc *ntb)
2223 if (!link_is_up(ntb))
2224 return (NTB_WIDTH_NONE);
2225 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta));
2228 SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0,
2229 "Driver state, statistics, and HW registers");
2231 #define NTB_REGSZ_MASK (3ul << 30)
2232 #define NTB_REG_64 (1ul << 30)
2233 #define NTB_REG_32 (2ul << 30)
2234 #define NTB_REG_16 (3ul << 30)
2235 #define NTB_REG_8 (0ul << 30)
2237 #define NTB_DB_READ (1ul << 29)
2238 #define NTB_PCI_REG (1ul << 28)
2239 #define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG)
2242 intel_ntb_sysctl_init(struct ntb_softc *ntb)
2244 struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar;
2245 struct sysctl_ctx_list *ctx;
2246 struct sysctl_oid *tree, *tmptree;
2248 ctx = device_get_sysctl_ctx(ntb->device);
2249 globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device));
2251 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status",
2252 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0,
2253 sysctl_handle_link_status_human, "A",
2254 "Link status (human readable)");
2255 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active",
2256 CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status,
2257 "IU", "Link status (1=active, 0=inactive)");
2258 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up",
2259 CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin,
2260 "IU", "Set/get interface status (1=UP, 0=DOWN)");
2262 tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info",
2263 CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers");
2264 tree_par = SYSCTL_CHILDREN(tree);
2266 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD,
2267 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port");
2268 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD,
2269 &ntb->dev_type, 0, "0 - USD; 1 - DSD");
2270 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD,
2271 &ntb->ppd, 0, "Raw PPD register (cached)");
2273 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) {
2274 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD,
2275 &ntb->b2b_mw_idx, 0,
2276 "Index of the MW used for B2B remote register access");
2277 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off",
2278 CTLFLAG_RD, &ntb->b2b_off,
2279 "If non-zero, offset of B2B register region in shared MW");
2282 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features",
2283 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A",
2284 "Features/errata of this NTB device");
2286 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD,
2287 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0,
2288 "NTB CTL register (cached)");
2289 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD,
2290 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0,
2291 "LNK STA register (cached)");
2293 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD,
2294 &ntb->mw_count, 0, "MW count");
2295 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD,
2296 &ntb->spad_count, 0, "Scratchpad count");
2297 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD,
2298 &ntb->db_count, 0, "Doorbell count");
2299 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD,
2300 &ntb->db_vec_count, 0, "Doorbell vector count");
2301 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD,
2302 &ntb->db_vec_shift, 0, "Doorbell vector shift");
2304 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD,
2305 &ntb->db_valid_mask, "Doorbell valid mask");
2306 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD,
2307 &ntb->db_link_mask, "Doorbell link mask");
2308 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD,
2309 &ntb->db_mask, "Doorbell mask (cached)");
2311 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers",
2312 CTLFLAG_RD, NULL, "Raw HW registers (big-endian)");
2313 regpar = SYSCTL_CHILDREN(tmptree);
2315 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl",
2316 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 |
2317 ntb->reg->ntb_ctl, sysctl_handle_register, "IU",
2318 "NTB Control register");
2319 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap",
2320 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 |
2321 0x19c, sysctl_handle_register, "IU",
2322 "NTB Link Capabilities");
2323 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon",
2324 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 |
2325 0x1a0, sysctl_handle_register, "IU",
2326 "NTB Link Control register");
2328 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask",
2329 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2330 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask,
2331 sysctl_handle_register, "QU", "Doorbell mask register");
2332 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell",
2333 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2334 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell,
2335 sysctl_handle_register, "QU", "Doorbell register");
2337 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23",
2338 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2339 NTB_REG_64 | ntb->xlat_reg->bar2_xlat,
2340 sysctl_handle_register, "QU", "Incoming XLAT23 register");
2341 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
2342 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4",
2343 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2344 NTB_REG_32 | ntb->xlat_reg->bar4_xlat,
2345 sysctl_handle_register, "IU", "Incoming XLAT4 register");
2346 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5",
2347 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2348 NTB_REG_32 | ntb->xlat_reg->bar5_xlat,
2349 sysctl_handle_register, "IU", "Incoming XLAT5 register");
2351 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45",
2352 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2353 NTB_REG_64 | ntb->xlat_reg->bar4_xlat,
2354 sysctl_handle_register, "QU", "Incoming XLAT45 register");
2357 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23",
2358 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2359 NTB_REG_64 | ntb->xlat_reg->bar2_limit,
2360 sysctl_handle_register, "QU", "Incoming LMT23 register");
2361 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
2362 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4",
2363 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2364 NTB_REG_32 | ntb->xlat_reg->bar4_limit,
2365 sysctl_handle_register, "IU", "Incoming LMT4 register");
2366 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5",
2367 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2368 NTB_REG_32 | ntb->xlat_reg->bar5_limit,
2369 sysctl_handle_register, "IU", "Incoming LMT5 register");
2371 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45",
2372 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2373 NTB_REG_64 | ntb->xlat_reg->bar4_limit,
2374 sysctl_handle_register, "QU", "Incoming LMT45 register");
2377 if (ntb->type == NTB_ATOM)
2380 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats",
2381 CTLFLAG_RD, NULL, "Xeon HW statistics");
2382 statpar = SYSCTL_CHILDREN(tmptree);
2383 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss",
2384 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2385 NTB_REG_16 | XEON_USMEMMISS_OFFSET,
2386 sysctl_handle_register, "SU", "Upstream Memory Miss");
2388 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err",
2389 CTLFLAG_RD, NULL, "Xeon HW errors");
2390 errpar = SYSCTL_CHILDREN(tmptree);
2392 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd",
2393 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2394 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET,
2395 sysctl_handle_register, "CU", "PPD");
2397 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz",
2398 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2399 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET,
2400 sysctl_handle_register, "CU", "PBAR23 SZ (log2)");
2401 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz",
2402 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2403 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET,
2404 sysctl_handle_register, "CU", "PBAR4 SZ (log2)");
2405 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz",
2406 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2407 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET,
2408 sysctl_handle_register, "CU", "PBAR5 SZ (log2)");
2410 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz",
2411 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2412 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET,
2413 sysctl_handle_register, "CU", "SBAR23 SZ (log2)");
2414 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz",
2415 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2416 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET,
2417 sysctl_handle_register, "CU", "SBAR4 SZ (log2)");
2418 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz",
2419 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2420 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET,
2421 sysctl_handle_register, "CU", "SBAR5 SZ (log2)");
2423 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts",
2424 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2425 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET,
2426 sysctl_handle_register, "SU", "DEVSTS");
2427 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts",
2428 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2429 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET,
2430 sysctl_handle_register, "SU", "LNKSTS");
2431 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts",
2432 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2433 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET,
2434 sysctl_handle_register, "SU", "SLNKSTS");
2436 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts",
2437 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2438 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET,
2439 sysctl_handle_register, "IU", "UNCERRSTS");
2440 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts",
2441 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2442 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET,
2443 sysctl_handle_register, "IU", "CORERRSTS");
2445 if (ntb->conn_type != NTB_CONN_B2B)
2448 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat01l",
2449 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2450 NTB_REG_32 | XEON_B2B_XLAT_OFFSETL,
2451 sysctl_handle_register, "IU", "Outgoing XLAT0L register");
2452 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat01u",
2453 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2454 NTB_REG_32 | XEON_B2B_XLAT_OFFSETU,
2455 sysctl_handle_register, "IU", "Outgoing XLAT0U register");
2456 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23",
2457 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2458 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off,
2459 sysctl_handle_register, "QU", "Outgoing XLAT23 register");
2460 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
2461 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4",
2462 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2463 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off,
2464 sysctl_handle_register, "IU", "Outgoing XLAT4 register");
2465 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5",
2466 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2467 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off,
2468 sysctl_handle_register, "IU", "Outgoing XLAT5 register");
2470 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45",
2471 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2472 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off,
2473 sysctl_handle_register, "QU", "Outgoing XLAT45 register");
2476 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23",
2477 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2478 NTB_REG_64 | XEON_PBAR2LMT_OFFSET,
2479 sysctl_handle_register, "QU", "Outgoing LMT23 register");
2480 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
2481 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4",
2482 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2483 NTB_REG_32 | XEON_PBAR4LMT_OFFSET,
2484 sysctl_handle_register, "IU", "Outgoing LMT4 register");
2485 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5",
2486 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2487 NTB_REG_32 | XEON_PBAR5LMT_OFFSET,
2488 sysctl_handle_register, "IU", "Outgoing LMT5 register");
2490 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45",
2491 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2492 NTB_REG_64 | XEON_PBAR4LMT_OFFSET,
2493 sysctl_handle_register, "QU", "Outgoing LMT45 register");
2496 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base",
2497 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2498 NTB_REG_64 | ntb->xlat_reg->bar0_base,
2499 sysctl_handle_register, "QU", "Secondary BAR01 base register");
2500 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base",
2501 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2502 NTB_REG_64 | ntb->xlat_reg->bar2_base,
2503 sysctl_handle_register, "QU", "Secondary BAR23 base register");
2504 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) {
2505 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base",
2506 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2507 NTB_REG_32 | ntb->xlat_reg->bar4_base,
2508 sysctl_handle_register, "IU",
2509 "Secondary BAR4 base register");
2510 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base",
2511 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2512 NTB_REG_32 | ntb->xlat_reg->bar5_base,
2513 sysctl_handle_register, "IU",
2514 "Secondary BAR5 base register");
2516 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base",
2517 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb,
2518 NTB_REG_64 | ntb->xlat_reg->bar4_base,
2519 sysctl_handle_register, "QU",
2520 "Secondary BAR45 base register");
2525 sysctl_handle_features(SYSCTL_HANDLER_ARGS)
2527 struct ntb_softc *ntb = arg1;
2531 sbuf_new_for_sysctl(&sb, NULL, 256, req);
2533 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR);
2534 error = sbuf_finish(&sb);
2537 if (error || !req->newptr)
2543 sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS)
2545 struct ntb_softc *ntb = arg1;
2549 old = intel_ntb_link_enabled(ntb->device);
2551 error = SYSCTL_OUT(req, &old, sizeof(old));
2552 if (error != 0 || req->newptr == NULL)
2555 error = SYSCTL_IN(req, &new, sizeof(new));
2559 intel_ntb_printf(0, "Admin set interface state to '%sabled'\n",
2560 (new != 0)? "en" : "dis");
2563 error = intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
2565 error = intel_ntb_link_disable(ntb->device);
2570 sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS)
2572 struct ntb_softc *ntb = arg1;
2574 enum ntb_speed speed;
2575 enum ntb_width width;
2578 sbuf_new_for_sysctl(&sb, NULL, 32, req);
2580 if (intel_ntb_link_is_up(ntb->device, &speed, &width))
2581 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u",
2582 (unsigned)speed, (unsigned)width);
2584 sbuf_printf(&sb, "down");
2586 error = sbuf_finish(&sb);
2589 if (error || !req->newptr)
2595 sysctl_handle_link_status(SYSCTL_HANDLER_ARGS)
2597 struct ntb_softc *ntb = arg1;
2601 res = intel_ntb_link_is_up(ntb->device, NULL, NULL);
2603 error = SYSCTL_OUT(req, &res, sizeof(res));
2604 if (error || !req->newptr)
2610 sysctl_handle_register(SYSCTL_HANDLER_ARGS)
2612 struct ntb_softc *ntb;
2616 char be[sizeof(umv)];
2623 reg = arg2 & ~NTB_REGFLAGS_MASK;
2624 sz = arg2 & NTB_REGSZ_MASK;
2625 db = (arg2 & NTB_DB_READ) != 0;
2626 pci = (arg2 & NTB_PCI_REG) != 0;
2628 KASSERT(!(db && pci), ("bogus"));
2631 KASSERT(sz == NTB_REG_64, ("bogus"));
2632 umv = db_ioread(ntb, reg);
2633 outsz = sizeof(uint64_t);
2638 umv = pci_read_config(ntb->device, reg, 8);
2640 umv = intel_ntb_reg_read(8, reg);
2641 outsz = sizeof(uint64_t);
2645 umv = pci_read_config(ntb->device, reg, 4);
2647 umv = intel_ntb_reg_read(4, reg);
2648 outsz = sizeof(uint32_t);
2652 umv = pci_read_config(ntb->device, reg, 2);
2654 umv = intel_ntb_reg_read(2, reg);
2655 outsz = sizeof(uint16_t);
2659 umv = pci_read_config(ntb->device, reg, 1);
2661 umv = intel_ntb_reg_read(1, reg);
2662 outsz = sizeof(uint8_t);
2670 /* Encode bigendian so that sysctl -x is legible. */
2672 outp = ((char *)be) + sizeof(umv) - outsz;
2674 error = SYSCTL_OUT(req, outp, outsz);
2675 if (error || !req->newptr)
2681 intel_ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx)
2684 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 &&
2685 uidx >= ntb->b2b_mw_idx) ||
2686 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx))
2688 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 &&
2689 uidx >= ntb->b2b_mw_idx) &&
2690 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx))
2695 #ifndef EARLY_AP_STARTUP
2696 static int msix_ready;
2699 intel_ntb_msix_ready(void *arg __unused)
2704 SYSINIT(intel_ntb_msix_ready, SI_SUB_SMP, SI_ORDER_ANY,
2705 intel_ntb_msix_ready, NULL);
2709 intel_ntb_exchange_msix(void *ctx)
2711 struct ntb_softc *ntb;
2717 if (ntb->peer_msix_good)
2719 if (ntb->peer_msix_done)
2722 #ifndef EARLY_AP_STARTUP
2723 /* Block MSIX negotiation until SMP started and IRQ reshuffled. */
2728 intel_ntb_get_msix_info(ntb);
2729 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
2730 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i,
2731 ntb->msix_data[i].nmd_data);
2732 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i,
2733 ntb->msix_data[i].nmd_ofs - ntb->msix_xlat);
2735 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD);
2737 intel_ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val);
2738 if (val != NTB_MSIX_VER_GUARD)
2741 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
2742 intel_ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val);
2743 intel_ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val);
2744 ntb->peer_msix_data[i].nmd_data = val;
2745 intel_ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val);
2746 intel_ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val);
2747 ntb->peer_msix_data[i].nmd_ofs = val;
2750 ntb->peer_msix_done = true;
2753 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED);
2754 intel_ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val);
2755 if (val != NTB_MSIX_RECEIVED)
2758 intel_ntb_spad_clear(ntb->device);
2759 ntb->peer_msix_good = true;
2760 /* Give peer time to see our NTB_MSIX_RECEIVED. */
2764 intel_ntb_poll_link(ntb);
2765 ntb_link_event(ntb->device);
2769 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
2770 if (_xeon_link_is_up(ntb)) {
2771 callout_reset(&ntb->peer_msix_work,
2772 hz * (ntb->peer_msix_good ? 2 : 1) / 10,
2773 intel_ntb_exchange_msix, ntb);
2775 intel_ntb_spad_clear(ntb->device);
2779 * Public API to the rest of the OS
2783 intel_ntb_spad_count(device_t dev)
2785 struct ntb_softc *ntb = device_get_softc(dev);
2787 return (ntb->spad_count);
2791 intel_ntb_mw_count(device_t dev)
2793 struct ntb_softc *ntb = device_get_softc(dev);
2796 res = ntb->mw_count;
2797 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0)
2799 if (ntb->msix_mw_idx != B2B_MW_DISABLED)
2805 intel_ntb_spad_write(device_t dev, unsigned int idx, uint32_t val)
2807 struct ntb_softc *ntb = device_get_softc(dev);
2809 if (idx >= ntb->spad_count)
2812 intel_ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val);
2818 * Zeros the local scratchpad.
2821 intel_ntb_spad_clear(device_t dev)
2823 struct ntb_softc *ntb = device_get_softc(dev);
2826 for (i = 0; i < ntb->spad_count; i++)
2827 intel_ntb_spad_write(dev, i, 0);
2831 intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val)
2833 struct ntb_softc *ntb = device_get_softc(dev);
2835 if (idx >= ntb->spad_count)
2838 *val = intel_ntb_reg_read(4, ntb->self_reg->spad + idx * 4);
2844 intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val)
2846 struct ntb_softc *ntb = device_get_softc(dev);
2848 if (idx >= ntb->spad_count)
2851 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP))
2852 intel_ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val);
2854 intel_ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val);
2860 intel_ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val)
2862 struct ntb_softc *ntb = device_get_softc(dev);
2864 if (idx >= ntb->spad_count)
2867 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP))
2868 *val = intel_ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4);
2870 *val = intel_ntb_reg_read(4, ntb->peer_reg->spad + idx * 4);
2876 intel_ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base,
2877 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size,
2880 struct ntb_softc *ntb = device_get_softc(dev);
2881 struct ntb_pci_bar_info *bar;
2884 enum ntb_bar bar_num;
2886 if (mw_idx >= intel_ntb_mw_count(dev))
2888 mw_idx = intel_ntb_user_mw_to_idx(ntb, mw_idx);
2890 bar_num = intel_ntb_mw_to_bar(ntb, mw_idx);
2891 bar = &ntb->bar_info[bar_num];
2893 if (mw_idx == ntb->b2b_mw_idx) {
2894 KASSERT(ntb->b2b_off != 0,
2895 ("user shouldn't get non-shared b2b mw"));
2896 bar_b2b_off = ntb->b2b_off;
2899 if (bar_is_64bit(ntb, bar_num))
2900 limit = BUS_SPACE_MAXADDR;
2902 limit = BUS_SPACE_MAXADDR_32BIT;
2905 *base = bar->pbase + bar_b2b_off;
2907 *vbase = bar->vbase + bar_b2b_off;
2909 *size = bar->size - bar_b2b_off;
2912 if (align_size != NULL)
2920 intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size)
2922 struct ntb_softc *ntb = device_get_softc(dev);
2923 struct ntb_pci_bar_info *bar;
2924 uint64_t base, limit, reg_val;
2925 size_t bar_size, mw_size;
2926 uint32_t base_reg, xlat_reg, limit_reg;
2927 enum ntb_bar bar_num;
2929 if (idx >= intel_ntb_mw_count(dev))
2931 idx = intel_ntb_user_mw_to_idx(ntb, idx);
2933 bar_num = intel_ntb_mw_to_bar(ntb, idx);
2934 bar = &ntb->bar_info[bar_num];
2936 bar_size = bar->size;
2937 if (idx == ntb->b2b_mw_idx)
2938 mw_size = bar_size - ntb->b2b_off;
2942 /* Hardware requires that addr is aligned to bar size */
2943 if ((addr & (bar_size - 1)) != 0)
2949 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg);
2952 if (bar_is_64bit(ntb, bar_num)) {
2953 base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK;
2955 if (limit_reg != 0 && size != mw_size)
2956 limit = base + size;
2958 /* Set and verify translation address */
2959 intel_ntb_reg_write(8, xlat_reg, addr);
2960 reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK;
2961 if (reg_val != addr) {
2962 intel_ntb_reg_write(8, xlat_reg, 0);
2966 /* Set and verify the limit */
2967 intel_ntb_reg_write(8, limit_reg, limit);
2968 reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK;
2969 if (reg_val != limit) {
2970 intel_ntb_reg_write(8, limit_reg, base);
2971 intel_ntb_reg_write(8, xlat_reg, 0);
2975 /* Configure 32-bit (split) BAR MW */
2977 if ((addr & UINT32_MAX) != addr)
2979 if (((addr + size) & UINT32_MAX) != (addr + size))
2982 base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK;
2984 if (limit_reg != 0 && size != mw_size)
2985 limit = base + size;
2987 /* Set and verify translation address */
2988 intel_ntb_reg_write(4, xlat_reg, addr);
2989 reg_val = intel_ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK;
2990 if (reg_val != addr) {
2991 intel_ntb_reg_write(4, xlat_reg, 0);
2995 /* Set and verify the limit */
2996 intel_ntb_reg_write(4, limit_reg, limit);
2997 reg_val = intel_ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK;
2998 if (reg_val != limit) {
2999 intel_ntb_reg_write(4, limit_reg, base);
3000 intel_ntb_reg_write(4, xlat_reg, 0);
3008 intel_ntb_mw_clear_trans(device_t dev, unsigned mw_idx)
3011 return (intel_ntb_mw_set_trans(dev, mw_idx, 0, 0));
3015 intel_ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode)
3017 struct ntb_softc *ntb = device_get_softc(dev);
3018 struct ntb_pci_bar_info *bar;
3020 if (idx >= intel_ntb_mw_count(dev))
3022 idx = intel_ntb_user_mw_to_idx(ntb, idx);
3024 bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)];
3025 *mode = bar->map_mode;
3030 intel_ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode)
3032 struct ntb_softc *ntb = device_get_softc(dev);
3034 if (idx >= intel_ntb_mw_count(dev))
3037 idx = intel_ntb_user_mw_to_idx(ntb, idx);
3038 return (intel_ntb_mw_set_wc_internal(ntb, idx, mode));
3042 intel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode)
3044 struct ntb_pci_bar_info *bar;
3047 bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)];
3048 if (bar->map_mode == mode)
3051 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode);
3053 bar->map_mode = mode;
3059 intel_ntb_peer_db_set(device_t dev, uint64_t bit)
3061 struct ntb_softc *ntb = device_get_softc(dev);
3063 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
3064 struct ntb_pci_bar_info *lapic;
3067 lapic = ntb->peer_lapic_bar;
3069 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
3070 if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0)
3071 bus_space_write_4(lapic->pci_bus_tag,
3072 lapic->pci_bus_handle,
3073 ntb->peer_msix_data[i].nmd_ofs,
3074 ntb->peer_msix_data[i].nmd_data);
3079 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
3080 intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit);
3084 db_iowrite(ntb, ntb->peer_reg->db_bell, bit);
3088 intel_ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size)
3090 struct ntb_softc *ntb = device_get_softc(dev);
3091 struct ntb_pci_bar_info *bar;
3094 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL"));
3096 if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
3097 bar = &ntb->bar_info[NTB_CONFIG_BAR];
3098 regoff = ntb->peer_reg->db_bell;
3100 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED,
3101 ("invalid b2b idx"));
3103 bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)];
3104 regoff = XEON_PDOORBELL_OFFSET;
3106 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh"));
3108 /* HACK: Specific to current x86 bus implementation. */
3109 *db_addr = ((uint64_t)bar->pci_bus_handle + regoff);
3110 *db_size = ntb->reg->db_size;
3115 intel_ntb_db_valid_mask(device_t dev)
3117 struct ntb_softc *ntb = device_get_softc(dev);
3119 return (ntb->db_valid_mask);
3123 intel_ntb_db_vector_count(device_t dev)
3125 struct ntb_softc *ntb = device_get_softc(dev);
3127 return (ntb->db_vec_count);
3131 intel_ntb_db_vector_mask(device_t dev, uint32_t vector)
3133 struct ntb_softc *ntb = device_get_softc(dev);
3135 if (vector > ntb->db_vec_count)
3137 return (ntb->db_valid_mask & intel_ntb_vec_mask(ntb, vector));
3141 intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width)
3143 struct ntb_softc *ntb = device_get_softc(dev);
3146 *speed = intel_ntb_link_sta_speed(ntb);
3148 *width = intel_ntb_link_sta_width(ntb);
3149 return (link_is_up(ntb));
3153 save_bar_parameters(struct ntb_pci_bar_info *bar)
3156 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource);
3157 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource);
3158 bar->pbase = rman_get_start(bar->pci_resource);
3159 bar->size = rman_get_size(bar->pci_resource);
3160 bar->vbase = rman_get_virtual(bar->pci_resource);
3163 static device_method_t ntb_intel_methods[] = {
3164 /* Device interface */
3165 DEVMETHOD(device_probe, intel_ntb_probe),
3166 DEVMETHOD(device_attach, intel_ntb_attach),
3167 DEVMETHOD(device_detach, intel_ntb_detach),
3169 DEVMETHOD(bus_child_location_str, ntb_child_location_str),
3170 DEVMETHOD(bus_print_child, ntb_print_child),
3171 DEVMETHOD(bus_get_dma_tag, ntb_get_dma_tag),
3173 DEVMETHOD(ntb_port_number, intel_ntb_port_number),
3174 DEVMETHOD(ntb_peer_port_count, intel_ntb_peer_port_count),
3175 DEVMETHOD(ntb_peer_port_number, intel_ntb_peer_port_number),
3176 DEVMETHOD(ntb_peer_port_idx, intel_ntb_peer_port_idx),
3177 DEVMETHOD(ntb_link_is_up, intel_ntb_link_is_up),
3178 DEVMETHOD(ntb_link_enable, intel_ntb_link_enable),
3179 DEVMETHOD(ntb_link_disable, intel_ntb_link_disable),
3180 DEVMETHOD(ntb_link_enabled, intel_ntb_link_enabled),
3181 DEVMETHOD(ntb_mw_count, intel_ntb_mw_count),
3182 DEVMETHOD(ntb_mw_get_range, intel_ntb_mw_get_range),
3183 DEVMETHOD(ntb_mw_set_trans, intel_ntb_mw_set_trans),
3184 DEVMETHOD(ntb_mw_clear_trans, intel_ntb_mw_clear_trans),
3185 DEVMETHOD(ntb_mw_get_wc, intel_ntb_mw_get_wc),
3186 DEVMETHOD(ntb_mw_set_wc, intel_ntb_mw_set_wc),
3187 DEVMETHOD(ntb_spad_count, intel_ntb_spad_count),
3188 DEVMETHOD(ntb_spad_clear, intel_ntb_spad_clear),
3189 DEVMETHOD(ntb_spad_write, intel_ntb_spad_write),
3190 DEVMETHOD(ntb_spad_read, intel_ntb_spad_read),
3191 DEVMETHOD(ntb_peer_spad_write, intel_ntb_peer_spad_write),
3192 DEVMETHOD(ntb_peer_spad_read, intel_ntb_peer_spad_read),
3193 DEVMETHOD(ntb_db_valid_mask, intel_ntb_db_valid_mask),
3194 DEVMETHOD(ntb_db_vector_count, intel_ntb_db_vector_count),
3195 DEVMETHOD(ntb_db_vector_mask, intel_ntb_db_vector_mask),
3196 DEVMETHOD(ntb_db_clear, intel_ntb_db_clear),
3197 DEVMETHOD(ntb_db_clear_mask, intel_ntb_db_clear_mask),
3198 DEVMETHOD(ntb_db_read, intel_ntb_db_read),
3199 DEVMETHOD(ntb_db_set_mask, intel_ntb_db_set_mask),
3200 DEVMETHOD(ntb_peer_db_addr, intel_ntb_peer_db_addr),
3201 DEVMETHOD(ntb_peer_db_set, intel_ntb_peer_db_set),
3205 static DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods,
3206 sizeof(struct ntb_softc));
3207 DRIVER_MODULE(ntb_hw_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL);
3208 MODULE_DEPEND(ntb_hw_intel, ntb, 1, 1, 1);
3209 MODULE_VERSION(ntb_hw_intel, 1);
3210 MODULE_PNP_INFO("W32:vendor/device;D:#", pci, ntb_hw_intel, pci_ids,