2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2015 Netflix, Inc.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer,
12 * without modification, immediately at the beginning of the file.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 * derived from ata_xpt.c: Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
34 #include <sys/param.h>
36 #include <sys/endian.h>
37 #include <sys/systm.h>
38 #include <sys/types.h>
39 #include <sys/malloc.h>
40 #include <sys/kernel.h>
43 #include <sys/fcntl.h>
44 #include <sys/interrupt.h>
48 #include <sys/mutex.h>
49 #include <sys/sysctl.h>
52 #include <cam/cam_ccb.h>
53 #include <cam/cam_queue.h>
54 #include <cam/cam_periph.h>
55 #include <cam/cam_sim.h>
56 #include <cam/cam_xpt.h>
57 #include <cam/cam_xpt_sim.h>
58 #include <cam/cam_xpt_periph.h>
59 #include <cam/cam_xpt_internal.h>
60 #include <cam/cam_debug.h>
62 #include <cam/scsi/scsi_all.h>
63 #include <cam/scsi/scsi_message.h>
64 #include <cam/nvme/nvme_all.h>
65 #include <machine/stdarg.h> /* for xpt_print below */
68 struct nvme_quirk_entry {
70 #define CAM_QUIRK_MAXTAGS 1
75 /* Not even sure why we need this */
76 static periph_init_t nvme_probe_periph_init;
78 static struct periph_driver nvme_probe_driver =
80 nvme_probe_periph_init, "nvme_probe",
81 TAILQ_HEAD_INITIALIZER(nvme_probe_driver.units), /* generation */ 0,
85 PERIPHDRIVER_DECLARE(nvme_probe, nvme_probe_driver);
94 static char *nvme_probe_action_text[] = {
95 "NVME_PROBE_IDENTIFY",
101 #define NVME_PROBE_SET_ACTION(softc, newaction) \
104 text = nvme_probe_action_text; \
105 CAM_DEBUG((softc)->periph->path, CAM_DEBUG_PROBE, \
106 ("Probe %s to %s\n", text[(softc)->action], \
107 text[(newaction)])); \
108 (softc)->action = (newaction); \
112 NVME_PROBE_NO_ANNOUNCE = 0x04
116 TAILQ_HEAD(, ccb_hdr) request_ccbs;
117 nvme_probe_action action;
118 nvme_probe_flags flags;
120 struct cam_periph *periph;
123 static struct nvme_quirk_entry nvme_quirk_table[] =
127 // T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
128 // /*vendor*/"*", /*product*/"*", /*revision*/"*"
130 .quirks = 0, .mintags = 0, .maxtags = 0
134 static const int nvme_quirk_table_size =
135 sizeof(nvme_quirk_table) / sizeof(*nvme_quirk_table);
137 static cam_status nvme_probe_register(struct cam_periph *periph,
139 static void nvme_probe_schedule(struct cam_periph *nvme_probe_periph);
140 static void nvme_probe_start(struct cam_periph *periph, union ccb *start_ccb);
141 static void nvme_probe_cleanup(struct cam_periph *periph);
142 //static void nvme_find_quirk(struct cam_ed *device);
143 static void nvme_scan_lun(struct cam_periph *periph,
144 struct cam_path *path, cam_flags flags,
146 static struct cam_ed *
147 nvme_alloc_device(struct cam_eb *bus, struct cam_et *target,
149 static void nvme_device_transport(struct cam_path *path);
150 static void nvme_dev_async(u_int32_t async_code,
152 struct cam_et *target,
153 struct cam_ed *device,
155 static void nvme_action(union ccb *start_ccb);
156 static void nvme_announce_periph(struct cam_periph *periph);
157 static void nvme_proto_announce(struct cam_ed *device);
158 static void nvme_proto_denounce(struct cam_ed *device);
159 static void nvme_proto_debug_out(union ccb *ccb);
161 static struct xpt_xport_ops nvme_xport_ops = {
162 .alloc_device = nvme_alloc_device,
163 .action = nvme_action,
164 .async = nvme_dev_async,
165 .announce = nvme_announce_periph,
167 #define NVME_XPT_XPORT(x, X) \
168 static struct xpt_xport nvme_xport_ ## x = { \
169 .xport = XPORT_ ## X, \
171 .ops = &nvme_xport_ops, \
173 CAM_XPT_XPORT(nvme_xport_ ## x);
175 NVME_XPT_XPORT(nvme, NVME);
177 #undef NVME_XPT_XPORT
179 static struct xpt_proto_ops nvme_proto_ops = {
180 .announce = nvme_proto_announce,
181 .denounce = nvme_proto_denounce,
182 .debug_out = nvme_proto_debug_out,
184 static struct xpt_proto nvme_proto = {
187 .ops = &nvme_proto_ops,
189 CAM_XPT_PROTO(nvme_proto);
192 nvme_probe_periph_init()
198 nvme_probe_register(struct cam_periph *periph, void *arg)
200 union ccb *request_ccb; /* CCB representing the probe request */
201 nvme_probe_softc *softc;
203 request_ccb = (union ccb *)arg;
204 if (request_ccb == NULL) {
205 printf("nvme_probe_register: no probe CCB, "
206 "can't register device\n");
207 return(CAM_REQ_CMP_ERR);
210 softc = (nvme_probe_softc *)malloc(sizeof(*softc), M_CAMXPT, M_ZERO | M_NOWAIT);
213 printf("nvme_probe_register: Unable to probe new device. "
214 "Unable to allocate softc\n");
215 return(CAM_REQ_CMP_ERR);
217 TAILQ_INIT(&softc->request_ccbs);
218 TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h,
221 periph->softc = softc;
222 softc->periph = periph;
223 softc->action = NVME_PROBE_INVALID;
224 if (cam_periph_acquire(periph) != 0)
225 return (CAM_REQ_CMP_ERR);
227 CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n"));
229 // nvme_device_transport(periph->path);
230 nvme_probe_schedule(periph);
236 nvme_probe_schedule(struct cam_periph *periph)
239 nvme_probe_softc *softc;
241 softc = (nvme_probe_softc *)periph->softc;
242 ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs);
244 NVME_PROBE_SET_ACTION(softc, NVME_PROBE_IDENTIFY);
246 if (ccb->crcn.flags & CAM_EXPECT_INQ_CHANGE)
247 softc->flags |= NVME_PROBE_NO_ANNOUNCE;
249 softc->flags &= ~NVME_PROBE_NO_ANNOUNCE;
251 xpt_schedule(periph, CAM_PRIORITY_XPT);
255 nvme_probe_start(struct cam_periph *periph, union ccb *start_ccb)
257 struct ccb_nvmeio *nvmeio;
258 struct ccb_scsiio *csio;
259 nvme_probe_softc *softc;
260 struct cam_path *path;
261 const struct nvme_namespace_data *nvme_data;
264 CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("nvme_probe_start\n"));
266 softc = (nvme_probe_softc *)periph->softc;
267 path = start_ccb->ccb_h.path;
268 nvmeio = &start_ccb->nvmeio;
269 csio = &start_ccb->csio;
270 nvme_data = periph->path->device->nvme_data;
272 if (softc->restart) {
274 if (periph->path->device->flags & CAM_DEV_UNCONFIGURED)
275 NVME_PROBE_SET_ACTION(softc, NVME_PROBE_RESET);
277 NVME_PROBE_SET_ACTION(softc, NVME_PROBE_IDENTIFY);
281 * Other transports have to ask their SIM to do a lot of action.
282 * NVMe doesn't, so don't do the dance. Just do things
285 switch (softc->action) {
286 case NVME_PROBE_RESET:
288 case NVME_PROBE_IDENTIFY:
289 nvme_device_transport(path);
291 * Test for lun == CAM_LUN_WILDCARD is lame, but
292 * appears to be necessary here. XXX
294 lun = xpt_path_lun_id(periph->path);
295 if (lun == CAM_LUN_WILDCARD ||
296 periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
297 path->device->flags &= ~CAM_DEV_UNCONFIGURED;
298 xpt_acquire_device(path->device);
299 start_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
300 xpt_action(start_ccb);
301 xpt_async(AC_FOUND_DEVICE, path, start_ccb);
303 NVME_PROBE_SET_ACTION(softc, NVME_PROBE_DONE);
306 panic("nvme_probe_start: invalid action state 0x%x\n", softc->action);
309 * Probing is now done. We need to complete any lingering items
310 * in the queue, though there shouldn't be any.
312 xpt_release_ccb(start_ccb);
313 CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe completed\n"));
314 while ((start_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs))) {
315 TAILQ_REMOVE(&softc->request_ccbs,
316 &start_ccb->ccb_h, periph_links.tqe);
317 start_ccb->ccb_h.status = CAM_REQ_CMP;
320 cam_periph_invalidate(periph);
321 cam_periph_release_locked(periph);
325 nvme_probe_cleanup(struct cam_periph *periph)
328 free(periph->softc, M_CAMXPT);
332 /* XXX should be used, don't delete */
334 nvme_find_quirk(struct cam_ed *device)
336 struct nvme_quirk_entry *quirk;
339 match = cam_quirkmatch((caddr_t)&device->nvme_data,
340 (caddr_t)nvme_quirk_table,
341 nvme_quirk_table_size,
342 sizeof(*nvme_quirk_table), nvme_identify_match);
345 panic("xpt_find_quirk: device didn't match wildcard entry!!");
347 quirk = (struct nvme_quirk_entry *)match;
348 device->quirk = quirk;
349 if (quirk->quirks & CAM_QUIRK_MAXTAGS) {
350 device->mintags = quirk->mintags;
351 device->maxtags = quirk->maxtags;
357 nvme_scan_lun(struct cam_periph *periph, struct cam_path *path,
358 cam_flags flags, union ccb *request_ccb)
360 struct ccb_pathinq cpi;
362 struct cam_periph *old_periph;
365 CAM_DEBUG(path, CAM_DEBUG_TRACE, ("nvme_scan_lun\n"));
367 xpt_path_inq(&cpi, path);
369 if (cpi.ccb_h.status != CAM_REQ_CMP) {
370 if (request_ccb != NULL) {
371 request_ccb->ccb_h.status = cpi.ccb_h.status;
372 xpt_done(request_ccb);
377 if (xpt_path_lun_id(path) == CAM_LUN_WILDCARD) {
378 CAM_DEBUG(path, CAM_DEBUG_TRACE, ("nvme_scan_lun ignoring bus\n"));
379 request_ccb->ccb_h.status = CAM_REQ_CMP; /* XXX signal error ? */
380 xpt_done(request_ccb);
384 lock = (xpt_path_owned(path) == 0);
387 if ((old_periph = cam_periph_find(path, "nvme_probe")) != NULL) {
388 if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) {
389 nvme_probe_softc *softc;
391 softc = (nvme_probe_softc *)old_periph->softc;
392 TAILQ_INSERT_TAIL(&softc->request_ccbs,
393 &request_ccb->ccb_h, periph_links.tqe);
395 CAM_DEBUG(path, CAM_DEBUG_TRACE,
396 ("restarting nvme_probe device\n"));
398 request_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
399 CAM_DEBUG(path, CAM_DEBUG_TRACE,
400 ("Failing to restart nvme_probe device\n"));
401 xpt_done(request_ccb);
404 CAM_DEBUG(path, CAM_DEBUG_TRACE,
405 ("Adding nvme_probe device\n"));
406 status = cam_periph_alloc(nvme_probe_register, NULL, nvme_probe_cleanup,
407 nvme_probe_start, "nvme_probe",
409 request_ccb->ccb_h.path, NULL, 0,
412 if (status != CAM_REQ_CMP) {
413 xpt_print(path, "xpt_scan_lun: cam_alloc_periph "
414 "returned an error, can't continue probe\n");
415 request_ccb->ccb_h.status = status;
416 xpt_done(request_ccb);
420 xpt_path_unlock(path);
423 static struct cam_ed *
424 nvme_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
426 struct nvme_quirk_entry *quirk;
427 struct cam_ed *device;
429 device = xpt_alloc_device(bus, target, lun_id);
434 * Take the default quirk entry until we have inquiry
435 * data from nvme and can determine a better quirk to use.
437 quirk = &nvme_quirk_table[nvme_quirk_table_size - 1];
438 device->quirk = (void *)quirk;
441 device->inq_flags = 0;
442 device->queue_flags = 0;
443 device->device_id = NULL; /* XXX Need to set this somewhere */
444 device->device_id_len = 0;
445 device->serial_num = NULL; /* XXX Need to set this somewhere */
446 device->serial_num_len = 0;
451 nvme_device_transport(struct cam_path *path)
453 struct ccb_pathinq cpi;
454 struct ccb_trans_settings cts;
455 /* XXX get data from nvme namespace and other info ??? */
457 /* Get transport information from the SIM */
458 xpt_path_inq(&cpi, path);
460 path->device->transport = cpi.transport;
461 path->device->transport_version = cpi.transport_version;
463 path->device->protocol = cpi.protocol;
464 path->device->protocol_version = cpi.protocol_version;
466 /* Tell the controller what we think */
467 xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
468 cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
469 cts.type = CTS_TYPE_CURRENT_SETTINGS;
470 cts.transport = path->device->transport;
471 cts.transport_version = path->device->transport_version;
472 cts.protocol = path->device->protocol;
473 cts.protocol_version = path->device->protocol_version;
474 cts.proto_specific.valid = 0;
475 cts.xport_specific.valid = 0;
476 xpt_action((union ccb *)&cts);
480 nvme_dev_advinfo(union ccb *start_ccb)
482 struct cam_ed *device;
483 struct ccb_dev_advinfo *cdai;
486 start_ccb->ccb_h.status = CAM_REQ_INVALID;
487 device = start_ccb->ccb_h.path->device;
488 cdai = &start_ccb->cdai;
489 switch(cdai->buftype) {
490 case CDAI_TYPE_SCSI_DEVID:
491 if (cdai->flags & CDAI_FLAG_STORE)
493 cdai->provsiz = device->device_id_len;
494 if (device->device_id_len == 0)
496 amt = device->device_id_len;
497 if (cdai->provsiz > cdai->bufsiz)
499 memcpy(cdai->buf, device->device_id, amt);
501 case CDAI_TYPE_SERIAL_NUM:
502 if (cdai->flags & CDAI_FLAG_STORE)
504 cdai->provsiz = device->serial_num_len;
505 if (device->serial_num_len == 0)
507 amt = device->serial_num_len;
508 if (cdai->provsiz > cdai->bufsiz)
510 memcpy(cdai->buf, device->serial_num, amt);
512 case CDAI_TYPE_PHYS_PATH:
513 if (cdai->flags & CDAI_FLAG_STORE) {
514 if (device->physpath != NULL)
515 free(device->physpath, M_CAMXPT);
516 device->physpath_len = cdai->bufsiz;
517 /* Clear existing buffer if zero length */
518 if (cdai->bufsiz == 0)
520 device->physpath = malloc(cdai->bufsiz, M_CAMXPT, M_NOWAIT);
521 if (device->physpath == NULL) {
522 start_ccb->ccb_h.status = CAM_REQ_ABORTED;
525 memcpy(device->physpath, cdai->buf, cdai->bufsiz);
527 cdai->provsiz = device->physpath_len;
528 if (device->physpath_len == 0)
530 amt = device->physpath_len;
531 if (cdai->provsiz > cdai->bufsiz)
533 memcpy(cdai->buf, device->physpath, amt);
536 case CDAI_TYPE_NVME_CNTRL:
537 if (cdai->flags & CDAI_FLAG_STORE)
539 amt = sizeof(struct nvme_controller_data);
541 if (amt > cdai->bufsiz)
543 memcpy(cdai->buf, device->nvme_cdata, amt);
545 case CDAI_TYPE_NVME_NS:
546 if (cdai->flags & CDAI_FLAG_STORE)
548 amt = sizeof(struct nvme_namespace_data);
550 if (amt > cdai->bufsiz)
552 memcpy(cdai->buf, device->nvme_data, amt);
557 start_ccb->ccb_h.status = CAM_REQ_CMP;
559 if (cdai->flags & CDAI_FLAG_STORE) {
560 xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path,
561 (void *)(uintptr_t)cdai->buftype);
566 nvme_action(union ccb *start_ccb)
568 CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE,
569 ("nvme_action: func= %#x\n", start_ccb->ccb_h.func_code));
571 switch (start_ccb->ccb_h.func_code) {
575 nvme_scan_lun(start_ccb->ccb_h.path->periph,
576 start_ccb->ccb_h.path, start_ccb->crcn.flags,
579 case XPT_DEV_ADVINFO:
580 nvme_dev_advinfo(start_ccb);
584 xpt_action_default(start_ccb);
590 * Handle any per-device event notifications that require action by the XPT.
593 nvme_dev_async(u_int32_t async_code, struct cam_eb *bus, struct cam_et *target,
594 struct cam_ed *device, void *async_arg)
598 * We only need to handle events for real devices.
600 if (target->target_id == CAM_TARGET_WILDCARD
601 || device->lun_id == CAM_LUN_WILDCARD)
604 if (async_code == AC_LOST_DEVICE &&
605 (device->flags & CAM_DEV_UNCONFIGURED) == 0) {
606 device->flags |= CAM_DEV_UNCONFIGURED;
607 xpt_release_device(device);
612 nvme_announce_periph(struct cam_periph *periph)
614 struct ccb_pathinq cpi;
615 struct ccb_trans_settings cts;
616 struct cam_path *path = periph->path;
617 struct ccb_trans_settings_nvme *nvmex;
619 cam_periph_assert(periph, MA_OWNED);
621 /* Ask the SIM for connection details */
622 xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
623 cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
624 cts.type = CTS_TYPE_CURRENT_SETTINGS;
625 xpt_action((union ccb*)&cts);
626 if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
628 nvmex = &cts.xport_specific.nvme;
630 /* Ask the SIM for its base transfer speed */
631 xpt_path_inq(&cpi, periph->path);
632 printf("%s%d: nvme version %d.%d x%d (max x%d) lanes PCIe Gen%d (max Gen%d) link",
633 periph->periph_name, periph->unit_number,
634 NVME_MAJOR(nvmex->spec),
635 NVME_MINOR(nvmex->spec),
636 nvmex->lanes, nvmex->max_lanes,
637 nvmex->speed, nvmex->max_speed);
642 nvme_proto_announce(struct cam_ed *device)
647 sbuf_new(&sb, buffer, sizeof(buffer), SBUF_FIXEDLEN);
648 nvme_print_ident(device->nvme_cdata, device->nvme_data, &sb);
654 nvme_proto_denounce(struct cam_ed *device)
657 nvme_proto_announce(device);
661 nvme_proto_debug_out(union ccb *ccb)
663 char cdb_str[(sizeof(struct nvme_command) * 3) + 1];
665 if (ccb->ccb_h.func_code != XPT_NVME_IO)
668 CAM_DEBUG(ccb->ccb_h.path,
669 CAM_DEBUG_CDB,("%s. NCB: %s\n", nvme_op_string(&ccb->nvmeio.cmd),
670 nvme_cmd_string(&ccb->nvmeio.cmd, cdb_str, sizeof(cdb_str))));