2 * Copyright (C) 2012 Intel Corporation
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
30 #include <sys/param.h>
33 #include <dev/pci/pcivar.h>
35 #include "nvme_private.h"
38 nvme_completion_check_retry(const struct nvme_completion *cpl)
41 * TODO: spec is not clear how commands that are aborted due
42 * to TLER will be marked. So for now, it seems
43 * NAMESPACE_NOT_READY is the only case where we should
44 * look at the DNR bit.
46 switch (cpl->sf_sct) {
47 case NVME_SCT_GENERIC:
49 case NVME_SC_NAMESPACE_NOT_READY:
54 case NVME_SC_INVALID_OPCODE:
55 case NVME_SC_INVALID_FIELD:
56 case NVME_SC_COMMAND_ID_CONFLICT:
57 case NVME_SC_DATA_TRANSFER_ERROR:
58 case NVME_SC_ABORTED_POWER_LOSS:
59 case NVME_SC_INTERNAL_DEVICE_ERROR:
60 case NVME_SC_ABORTED_BY_REQUEST:
61 case NVME_SC_ABORTED_SQ_DELETION:
62 case NVME_SC_ABORTED_FAILED_FUSED:
63 case NVME_SC_ABORTED_MISSING_FUSED:
64 case NVME_SC_INVALID_NAMESPACE_OR_FORMAT:
65 case NVME_SC_COMMAND_SEQUENCE_ERROR:
66 case NVME_SC_LBA_OUT_OF_RANGE:
67 case NVME_SC_CAPACITY_EXCEEDED:
71 case NVME_SCT_COMMAND_SPECIFIC:
72 case NVME_SCT_MEDIA_ERROR:
73 case NVME_SCT_VENDOR_SPECIFIC:
80 nvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
84 bus_dmamap_create(qpair->dma_tag, 0, &tr->payload_dma_map);
85 bus_dmamap_create(qpair->dma_tag, 0, &tr->prp_dma_map);
87 bus_dmamap_load(qpair->dma_tag, tr->prp_dma_map, tr->prp,
88 sizeof(tr->prp), nvme_single_map, &tr->prp_bus_addr, 0);
90 callout_init_mtx(&tr->timer, &qpair->lock, 0);
96 nvme_qpair_process_completions(struct nvme_qpair *qpair)
98 struct nvme_tracker *tr;
99 struct nvme_request *req;
100 struct nvme_completion *cpl;
101 boolean_t retry, error;
103 qpair->num_intr_handler_calls++;
106 cpl = &qpair->cpl[qpair->cq_head];
108 if (cpl->p != qpair->phase)
111 tr = qpair->act_tr[cpl->cid];
115 ("completion queue has entries but no active trackers\n"));
117 error = cpl->sf_sc || cpl->sf_sct;
118 retry = error && nvme_completion_check_retry(cpl);
121 nvme_dump_completion(cpl);
122 nvme_dump_command(&tr->req->cmd);
125 qpair->act_tr[cpl->cid] = NULL;
127 KASSERT(cpl->cid == req->cmd.cid,
128 ("cpl cid does not match cmd cid\n"));
130 if (req->cb_fn && !retry)
131 req->cb_fn(req->cb_arg, cpl);
133 qpair->sq_head = cpl->sqhd;
135 mtx_lock(&qpair->lock);
136 callout_stop(&tr->timer);
139 nvme_qpair_submit_cmd(qpair, tr);
141 if (req->payload_size > 0 || req->uio != NULL)
142 bus_dmamap_unload(qpair->dma_tag,
143 tr->payload_dma_map);
145 nvme_free_request(req);
147 SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist);
149 if (!STAILQ_EMPTY(&qpair->queued_req)) {
150 req = STAILQ_FIRST(&qpair->queued_req);
151 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
152 nvme_qpair_submit_request(qpair, req);
156 mtx_unlock(&qpair->lock);
158 if (++qpair->cq_head == qpair->num_entries) {
160 qpair->phase = !qpair->phase;
163 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].cq_hdbl,
169 nvme_qpair_msix_handler(void *arg)
171 struct nvme_qpair *qpair = arg;
173 nvme_qpair_process_completions(qpair);
177 nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
178 uint16_t vector, uint32_t num_entries, uint32_t num_trackers,
179 uint32_t max_xfer_size, struct nvme_controller *ctrlr)
181 struct nvme_tracker *tr;
185 qpair->vector = vector;
186 qpair->num_entries = num_entries;
189 * Chatham prototype board starts having issues at higher queue
190 * depths. So use a conservative estimate here of no more than 64
191 * outstanding I/O per queue at any one point.
193 if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID)
194 num_trackers = min(num_trackers, 64);
196 qpair->num_trackers = num_trackers;
197 qpair->max_xfer_size = max_xfer_size;
198 qpair->ctrlr = ctrlr;
201 * First time through the completion queue, HW will set phase
202 * bit on completions to 1. So set this to 1 here, indicating
203 * we're looking for a 1 to know which entries have completed.
204 * we'll toggle the bit each time when the completion queue
209 if (ctrlr->msix_enabled) {
212 * MSI-X vector resource IDs start at 1, so we add one to
213 * the queue's vector to get the corresponding rid to use.
215 qpair->rid = vector + 1;
217 qpair->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ,
218 &qpair->rid, RF_ACTIVE);
220 bus_setup_intr(ctrlr->dev, qpair->res,
221 INTR_TYPE_MISC | INTR_MPSAFE, NULL,
222 nvme_qpair_msix_handler, qpair, &qpair->tag);
225 mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF);
227 bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
228 sizeof(uint64_t), PAGE_SIZE, BUS_SPACE_MAXADDR,
229 BUS_SPACE_MAXADDR, NULL, NULL, qpair->max_xfer_size,
230 (qpair->max_xfer_size/PAGE_SIZE)+1, PAGE_SIZE, 0,
231 NULL, NULL, &qpair->dma_tag);
234 qpair->num_intr_handler_calls = 0;
235 qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0;
237 /* TODO: error checking on contigmalloc, bus_dmamap_load calls */
238 qpair->cmd = contigmalloc(qpair->num_entries *
239 sizeof(struct nvme_command), M_NVME, M_ZERO | M_NOWAIT,
240 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
241 qpair->cpl = contigmalloc(qpair->num_entries *
242 sizeof(struct nvme_completion), M_NVME, M_ZERO | M_NOWAIT,
243 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
245 bus_dmamap_create(qpair->dma_tag, 0, &qpair->cmd_dma_map);
246 bus_dmamap_create(qpair->dma_tag, 0, &qpair->cpl_dma_map);
248 bus_dmamap_load(qpair->dma_tag, qpair->cmd_dma_map,
249 qpair->cmd, qpair->num_entries * sizeof(struct nvme_command),
250 nvme_single_map, &qpair->cmd_bus_addr, 0);
251 bus_dmamap_load(qpair->dma_tag, qpair->cpl_dma_map,
252 qpair->cpl, qpair->num_entries * sizeof(struct nvme_completion),
253 nvme_single_map, &qpair->cpl_bus_addr, 0);
255 qpair->sq_tdbl_off = nvme_mmio_offsetof(doorbell[id].sq_tdbl);
256 qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl);
258 SLIST_INIT(&qpair->free_tr);
259 STAILQ_INIT(&qpair->queued_req);
261 for (i = 0; i < qpair->num_trackers; i++) {
262 tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_NOWAIT);
265 printf("warning: nvme tracker malloc failed\n");
269 nvme_qpair_construct_tracker(qpair, tr, i);
270 SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist);
273 qpair->act_tr = malloc(sizeof(struct nvme_tracker *) * qpair->num_entries,
274 M_NVME, M_ZERO | M_NOWAIT);
278 nvme_qpair_destroy(struct nvme_qpair *qpair)
280 struct nvme_tracker *tr;
283 bus_teardown_intr(qpair->ctrlr->dev, qpair->res, qpair->tag);
286 bus_release_resource(qpair->ctrlr->dev, SYS_RES_IRQ,
287 rman_get_rid(qpair->res), qpair->res);
290 bus_dma_tag_destroy(qpair->dma_tag);
293 free(qpair->act_tr, M_NVME);
295 while (!SLIST_EMPTY(&qpair->free_tr)) {
296 tr = SLIST_FIRST(&qpair->free_tr);
297 SLIST_REMOVE_HEAD(&qpair->free_tr, slist);
298 bus_dmamap_destroy(qpair->dma_tag, tr->payload_dma_map);
299 bus_dmamap_destroy(qpair->dma_tag, tr->prp_dma_map);
305 nvme_admin_qpair_destroy(struct nvme_qpair *qpair)
309 * For NVMe, you don't send delete queue commands for the admin
310 * queue, so we just need to unload and free the cmd and cpl memory.
312 bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map);
313 bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map);
315 contigfree(qpair->cmd,
316 qpair->num_entries * sizeof(struct nvme_command), M_NVME);
318 bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map);
319 bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map);
320 contigfree(qpair->cpl,
321 qpair->num_entries * sizeof(struct nvme_completion), M_NVME);
323 nvme_qpair_destroy(qpair);
327 nvme_free_cmd_ring(void *arg, const struct nvme_completion *status)
329 struct nvme_qpair *qpair;
331 qpair = (struct nvme_qpair *)arg;
332 bus_dmamap_unload(qpair->dma_tag, qpair->cmd_dma_map);
333 bus_dmamap_destroy(qpair->dma_tag, qpair->cmd_dma_map);
334 contigfree(qpair->cmd,
335 qpair->num_entries * sizeof(struct nvme_command), M_NVME);
340 nvme_free_cpl_ring(void *arg, const struct nvme_completion *status)
342 struct nvme_qpair *qpair;
344 qpair = (struct nvme_qpair *)arg;
345 bus_dmamap_unload(qpair->dma_tag, qpair->cpl_dma_map);
346 bus_dmamap_destroy(qpair->dma_tag, qpair->cpl_dma_map);
347 contigfree(qpair->cpl,
348 qpair->num_entries * sizeof(struct nvme_completion), M_NVME);
353 nvme_io_qpair_destroy(struct nvme_qpair *qpair)
355 struct nvme_controller *ctrlr = qpair->ctrlr;
357 if (qpair->num_entries > 0) {
359 nvme_ctrlr_cmd_delete_io_sq(ctrlr, qpair, nvme_free_cmd_ring,
361 /* Spin until free_cmd_ring sets qpair->cmd to NULL. */
365 nvme_ctrlr_cmd_delete_io_cq(ctrlr, qpair, nvme_free_cpl_ring,
367 /* Spin until free_cpl_ring sets qpair->cmd to NULL. */
371 nvme_qpair_destroy(qpair);
376 nvme_timeout(void *arg)
379 * TODO: Add explicit abort operation here, once nvme(4) supports
385 nvme_qpair_submit_cmd(struct nvme_qpair *qpair, struct nvme_tracker *tr)
387 struct nvme_request *req;
390 req->cmd.cid = tr->cid;
391 qpair->act_tr[tr->cid] = tr;
393 #if __FreeBSD_version >= 800030
394 callout_reset_curcpu(&tr->timer, NVME_TIMEOUT_IN_SEC * hz,
397 callout_reset(&tr->timer, NVME_TIMEOUT_IN_SEC * hz, nvme_timeout, tr);
400 /* Copy the command from the tracker to the submission queue. */
401 memcpy(&qpair->cmd[qpair->sq_tail], &req->cmd, sizeof(req->cmd));
403 if (++qpair->sq_tail == qpair->num_entries)
407 nvme_mmio_write_4(qpair->ctrlr, doorbell[qpair->id].sq_tdbl,
414 nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
416 struct nvme_tracker *tr;
419 mtx_lock(&qpair->lock);
421 tr = SLIST_FIRST(&qpair->free_tr);
425 * No tracker is available. Put the request on the qpair's
426 * request queue to be processed when a tracker frees up
427 * via a command completion.
429 STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq);
433 SLIST_REMOVE_HEAD(&qpair->free_tr, slist);
436 if (req->uio == NULL) {
437 if (req->payload_size > 0) {
438 err = bus_dmamap_load(tr->qpair->dma_tag,
439 tr->payload_dma_map, req->payload,
441 nvme_payload_map, tr, 0);
443 panic("bus_dmamap_load returned non-zero!\n");
445 nvme_qpair_submit_cmd(tr->qpair, tr);
447 err = bus_dmamap_load_uio(tr->qpair->dma_tag,
448 tr->payload_dma_map, req->uio,
449 nvme_payload_map_uio, tr, 0);
451 panic("bus_dmamap_load returned non-zero!\n");
455 mtx_unlock(&qpair->lock);