sys/dev/xdma/xdma_sg.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2018-2019 Ruslan Bukin <br@bsdpad.com>
   5  *
   6  * This software was developed by SRI International and the University of
   7  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
   8  * ("CTSRD"), as part of the DARPA CRASH research programme.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include "opt_platform.h"
  36 #include <sys/param.h>
  37 #include <sys/conf.h>
  38 #include <sys/bus.h>
  39 #include <sys/kernel.h>
  40 #include <sys/malloc.h>
  41 #include <sys/mbuf.h>
  42 #include <sys/rwlock.h>
  43
  44 #include <machine/bus.h>
  45
  46 #include <vm/vm.h>
  47 #include <vm/vm_extern.h>
  48 #include <vm/vm_page.h>
  49
  50 #ifdef FDT
  51 #include <dev/fdt/fdt_common.h>
  52 #include <dev/ofw/ofw_bus.h>
  53 #include <dev/ofw/ofw_bus_subr.h>
  54 #endif
  55
  56 #include <dev/xdma/xdma.h>
  57
  58 #include <xdma_if.h>
  59
  60 struct seg_load_request {
  61         struct bus_dma_segment *seg;
  62         uint32_t nsegs;
  63         uint32_t error;
  64 };
  65
  66 static void
  67 xchan_bufs_free_reserved(xdma_channel_t *xchan)
  68 {
  69         struct xdma_request *xr;
  70         vm_size_t size;
  71         int i;
  72
  73         for (i = 0; i < xchan->xr_num; i++) {
  74                 xr = &xchan->xr_mem[i];
  75                 size = xr->buf.size;
  76                 if (xr->buf.vaddr) {
  77                         pmap_kremove_device(xr->buf.vaddr, size);
  78                         kva_free(xr->buf.vaddr, size);
  79                         xr->buf.vaddr = 0;
  80                 }
  81                 if (xr->buf.paddr) {
  82                         vmem_free(xchan->vmem, xr->buf.paddr, size);
  83                         xr->buf.paddr = 0;
  84                 }
  85                 xr->buf.size = 0;
  86         }
  87 }
  88
  89 static int
  90 xchan_bufs_alloc_reserved(xdma_channel_t *xchan)
  91 {
  92         xdma_controller_t *xdma;
  93         struct xdma_request *xr;
  94         vmem_addr_t addr;
  95         vm_size_t size;
  96         int i;
  97
  98         xdma = xchan->xdma;
  99
 100         if (xchan->vmem == NULL)
 101                 return (ENOBUFS);
 102
 103         for (i = 0; i < xchan->xr_num; i++) {
 104                 xr = &xchan->xr_mem[i];
 105                 size = round_page(xchan->maxsegsize);
 106                 if (vmem_alloc(xchan->vmem, size,
 107                     M_BESTFIT | M_NOWAIT, &addr)) {
 108                         device_printf(xdma->dev,
 109                             "%s: Can't allocate memory\n", __func__);
 110                         xchan_bufs_free_reserved(xchan);
 111                         return (ENOMEM);
 112                 }
 113
 114                 xr->buf.size = size;
 115                 xr->buf.paddr = addr;
 116                 xr->buf.vaddr = kva_alloc(size);
 117                 if (xr->buf.vaddr == 0) {
 118                         device_printf(xdma->dev,
 119                             "%s: Can't allocate KVA\n", __func__);
 120                         xchan_bufs_free_reserved(xchan);
 121                         return (ENOMEM);
 122                 }
 123                 pmap_kenter_device(xr->buf.vaddr, size, addr);
 124         }
 125
 126         return (0);
 127 }
 128
 129 static int
 130 xchan_bufs_alloc_busdma(xdma_channel_t *xchan)
 131 {
 132         xdma_controller_t *xdma;
 133         struct xdma_request *xr;
 134         int err;
 135         int i;
 136
 137         xdma = xchan->xdma;
 138
 139         /* Create bus_dma tag */
 140         err = bus_dma_tag_create(
 141             bus_get_dma_tag(xdma->dev), /* Parent tag. */
 142             xchan->alignment,           /* alignment */
 143             xchan->boundary,            /* boundary */
 144             xchan->lowaddr,             /* lowaddr */
 145             xchan->highaddr,            /* highaddr */
 146             NULL, NULL,                 /* filter, filterarg */
 147             xchan->maxsegsize * xchan->maxnsegs, /* maxsize */
 148             xchan->maxnsegs,            /* nsegments */
 149             xchan->maxsegsize,          /* maxsegsize */
 150             0,                          /* flags */
 151             NULL, NULL,                 /* lockfunc, lockarg */
 152             &xchan->dma_tag_bufs);
 153         if (err != 0) {
 154                 device_printf(xdma->dev,
 155                     "%s: Can't create bus_dma tag.\n", __func__);
 156                 return (-1);
 157         }
 158
 159         for (i = 0; i < xchan->xr_num; i++) {
 160                 xr = &xchan->xr_mem[i];
 161                 err = bus_dmamap_create(xchan->dma_tag_bufs, 0,
 162                     &xr->buf.map);
 163                 if (err != 0) {
 164                         device_printf(xdma->dev,
 165                             "%s: Can't create buf DMA map.\n", __func__);
 166
 167                         /* Cleanup. */
 168                         bus_dma_tag_destroy(xchan->dma_tag_bufs);
 169
 170                         return (-1);
 171                 }
 172         }
 173
 174         return (0);
 175 }
 176
 177 static int
 178 xchan_bufs_alloc(xdma_channel_t *xchan)
 179 {
 180         xdma_controller_t *xdma;
 181         int ret;
 182
 183         xdma = xchan->xdma;
 184
 185         if (xdma == NULL) {
 186                 device_printf(xdma->dev,
 187                     "%s: Channel was not allocated properly.\n", __func__);
 188                 return (-1);
 189         }
 190
 191         if (xchan->caps & XCHAN_CAP_BUSDMA)
 192                 ret = xchan_bufs_alloc_busdma(xchan);
 193         else {
 194                 ret = xchan_bufs_alloc_reserved(xchan);
 195         }
 196         if (ret != 0) {
 197                 device_printf(xdma->dev,
 198                     "%s: Can't allocate bufs.\n", __func__);
 199                 return (-1);
 200         }
 201
 202         xchan->flags |= XCHAN_BUFS_ALLOCATED;
 203
 204         return (0);
 205 }
 206
 207 static int
 208 xchan_bufs_free(xdma_channel_t *xchan)
 209 {
 210         struct xdma_request *xr;
 211         struct xchan_buf *b;
 212         int i;
 213
 214         if ((xchan->flags & XCHAN_BUFS_ALLOCATED) == 0)
 215                 return (-1);
 216
 217         if (xchan->caps & XCHAN_CAP_BUSDMA) {
 218                 for (i = 0; i < xchan->xr_num; i++) {
 219                         xr = &xchan->xr_mem[i];
 220                         b = &xr->buf;
 221                         bus_dmamap_destroy(xchan->dma_tag_bufs, b->map);
 222                 }
 223                 bus_dma_tag_destroy(xchan->dma_tag_bufs);
 224         } else
 225                 xchan_bufs_free_reserved(xchan);
 226
 227         xchan->flags &= ~XCHAN_BUFS_ALLOCATED;
 228
 229         return (0);
 230 }
 231
 232 void
 233 xdma_channel_free_sg(xdma_channel_t *xchan)
 234 {
 235
 236         xchan_bufs_free(xchan);
 237         xchan_sglist_free(xchan);
 238         xchan_bank_free(xchan);
 239 }
 240
 241 /*
 242  * Prepare xchan for a scatter-gather transfer.
 243  * xr_num - xdma requests queue size,
 244  * maxsegsize - maximum allowed scatter-gather list element size in bytes
 245  */
 246 int
 247 xdma_prep_sg(xdma_channel_t *xchan, uint32_t xr_num,
 248     bus_size_t maxsegsize, bus_size_t maxnsegs,
 249     bus_size_t alignment, bus_addr_t boundary,
 250     bus_addr_t lowaddr, bus_addr_t highaddr)
 251 {
 252         xdma_controller_t *xdma;
 253         int ret;
 254
 255         xdma = xchan->xdma;
 256
 257         KASSERT(xdma != NULL, ("xdma is NULL"));
 258
 259         if (xchan->flags & XCHAN_CONFIGURED) {
 260                 device_printf(xdma->dev,
 261                     "%s: Channel is already configured.\n", __func__);
 262                 return (-1);
 263         }
 264
 265         xchan->xr_num = xr_num;
 266         xchan->maxsegsize = maxsegsize;
 267         xchan->maxnsegs = maxnsegs;
 268         xchan->alignment = alignment;
 269         xchan->boundary = boundary;
 270         xchan->lowaddr = lowaddr;
 271         xchan->highaddr = highaddr;
 272
 273         if (xchan->maxnsegs > XDMA_MAX_SEG) {
 274                 device_printf(xdma->dev, "%s: maxnsegs is too big\n",
 275                     __func__);
 276                 return (-1);
 277         }
 278
 279         xchan_bank_init(xchan);
 280
 281         /* Allocate sglist. */
 282         ret = xchan_sglist_alloc(xchan);
 283         if (ret != 0) {
 284                 device_printf(xdma->dev,
 285                     "%s: Can't allocate sglist.\n", __func__);
 286                 return (-1);
 287         }
 288
 289         /* Allocate buffers if required. */
 290         if ((xchan->caps & XCHAN_CAP_NOBUFS) == 0) {
 291                 ret = xchan_bufs_alloc(xchan);
 292                 if (ret != 0) {
 293                         device_printf(xdma->dev,
 294                             "%s: Can't allocate bufs.\n", __func__);
 295
 296                         /* Cleanup */
 297                         xchan_sglist_free(xchan);
 298                         xchan_bank_free(xchan);
 299
 300                         return (-1);
 301                 }
 302         }
 303
 304         xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_SG);
 305
 306         XCHAN_LOCK(xchan);
 307         ret = XDMA_CHANNEL_PREP_SG(xdma->dma_dev, xchan);
 308         if (ret != 0) {
 309                 device_printf(xdma->dev,
 310                     "%s: Can't prepare SG transfer.\n", __func__);
 311                 XCHAN_UNLOCK(xchan);
 312
 313                 return (-1);
 314         }
 315         XCHAN_UNLOCK(xchan);
 316
 317         return (0);
 318 }
 319
 320 void
 321 xchan_seg_done(xdma_channel_t *xchan,
 322     struct xdma_transfer_status *st)
 323 {
 324         struct xdma_request *xr;
 325         xdma_controller_t *xdma;
 326         struct xchan_buf *b;
 327
 328         xdma = xchan->xdma;
 329
 330         xr = TAILQ_FIRST(&xchan->processing);
 331         if (xr == NULL)
 332                 panic("request not found\n");
 333
 334         b = &xr->buf;
 335
 336         atomic_subtract_int(&b->nsegs_left, 1);
 337
 338         if (b->nsegs_left == 0) {
 339                 if (xchan->caps & XCHAN_CAP_BUSDMA) {
 340                         if (xr->direction == XDMA_MEM_TO_DEV)
 341                                 bus_dmamap_sync(xchan->dma_tag_bufs, b->map,
 342                                     BUS_DMASYNC_POSTWRITE);
 343                         else
 344                                 bus_dmamap_sync(xchan->dma_tag_bufs, b->map,
 345                                     BUS_DMASYNC_POSTREAD);
 346                         bus_dmamap_unload(xchan->dma_tag_bufs, b->map);
 347                 } else {
 348                         if (xr->req_type == XR_TYPE_MBUF &&
 349                             xr->direction == XDMA_DEV_TO_MEM)
 350                                 m_copyback(xr->m, 0, st->transferred,
 351                                     (void *)xr->buf.vaddr);
 352                 }
 353                 xr->status.error = st->error;
 354                 xr->status.transferred = st->transferred;
 355
 356                 QUEUE_PROC_LOCK(xchan);
 357                 TAILQ_REMOVE(&xchan->processing, xr, xr_next);
 358                 QUEUE_PROC_UNLOCK(xchan);
 359
 360                 QUEUE_OUT_LOCK(xchan);
 361                 TAILQ_INSERT_TAIL(&xchan->queue_out, xr, xr_next);
 362                 QUEUE_OUT_UNLOCK(xchan);
 363         }
 364 }
 365
 366 static void
 367 xdma_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 368 {
 369         struct seg_load_request *slr;
 370         struct bus_dma_segment *seg;
 371         int i;
 372
 373         slr = arg;
 374         seg = slr->seg;
 375
 376         if (error != 0) {
 377                 slr->error = error;
 378                 return;
 379         }
 380
 381         slr->nsegs = nsegs;
 382
 383         for (i = 0; i < nsegs; i++) {
 384                 seg[i].ds_addr = segs[i].ds_addr;
 385                 seg[i].ds_len = segs[i].ds_len;
 386         }
 387 }
 388
 389 static int
 390 _xdma_load_data_busdma(xdma_channel_t *xchan, struct xdma_request *xr,
 391     struct bus_dma_segment *seg)
 392 {
 393         xdma_controller_t *xdma;
 394         struct seg_load_request slr;
 395         uint32_t nsegs;
 396         void *addr;
 397         int error;
 398
 399         xdma = xchan->xdma;
 400
 401         error = 0;
 402         nsegs = 0;
 403
 404         switch (xr->req_type) {
 405         case XR_TYPE_MBUF:
 406                 error = bus_dmamap_load_mbuf_sg(xchan->dma_tag_bufs,
 407                     xr->buf.map, xr->m, seg, &nsegs, BUS_DMA_NOWAIT);
 408                 break;
 409         case XR_TYPE_BIO:
 410                 slr.nsegs = 0;
 411                 slr.error = 0;
 412                 slr.seg = seg;
 413                 error = bus_dmamap_load_bio(xchan->dma_tag_bufs,
 414                     xr->buf.map, xr->bp, xdma_dmamap_cb, &slr, BUS_DMA_NOWAIT);
 415                 if (slr.error != 0) {
 416                         device_printf(xdma->dma_dev,
 417                             "%s: bus_dmamap_load failed, err %d\n",
 418                             __func__, slr.error);
 419                         return (0);
 420                 }
 421                 nsegs = slr.nsegs;
 422                 break;
 423         case XR_TYPE_VIRT:
 424                 switch (xr->direction) {
 425                 case XDMA_MEM_TO_DEV:
 426                         addr = (void *)xr->src_addr;
 427                         break;
 428                 case XDMA_DEV_TO_MEM:
 429                         addr = (void *)xr->dst_addr;
 430                         break;
 431                 default:
 432                         device_printf(xdma->dma_dev,
 433                             "%s: Direction is not supported\n", __func__);
 434                         return (0);
 435                 }
 436                 slr.nsegs = 0;
 437                 slr.error = 0;
 438                 slr.seg = seg;
 439                 error = bus_dmamap_load(xchan->dma_tag_bufs, xr->buf.map,
 440                     addr, (xr->block_len * xr->block_num),
 441                     xdma_dmamap_cb, &slr, BUS_DMA_NOWAIT);
 442                 if (slr.error != 0) {
 443                         device_printf(xdma->dma_dev,
 444                             "%s: bus_dmamap_load failed, err %d\n",
 445                             __func__, slr.error);
 446                         return (0);
 447                 }
 448                 nsegs = slr.nsegs;
 449                 break;
 450         default:
 451                 break;
 452         }
 453
 454         if (error != 0) {
 455                 if (error == ENOMEM) {
 456                         /*
 457                          * Out of memory. Try again later.
 458                          * TODO: count errors.
 459                          */
 460                 } else
 461                         device_printf(xdma->dma_dev,
 462                             "%s: bus_dmamap_load failed with err %d\n",
 463                             __func__, error);
 464                 return (0);
 465         }
 466
 467         if (xr->direction == XDMA_MEM_TO_DEV)
 468                 bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map,
 469                     BUS_DMASYNC_PREWRITE);
 470         else
 471                 bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map,
 472                     BUS_DMASYNC_PREREAD);
 473
 474         return (nsegs);
 475 }
 476
 477 static int
 478 _xdma_load_data(xdma_channel_t *xchan, struct xdma_request *xr,
 479     struct bus_dma_segment *seg)
 480 {
 481         xdma_controller_t *xdma;
 482         struct mbuf *m;
 483         uint32_t nsegs;
 484
 485         xdma = xchan->xdma;
 486
 487         m = xr->m;
 488
 489         nsegs = 1;
 490
 491         switch (xr->req_type) {
 492         case XR_TYPE_MBUF:
 493                 if ((xchan->caps & XCHAN_CAP_NOBUFS) == 0) {
 494                         if (xr->direction == XDMA_MEM_TO_DEV)
 495                                 m_copydata(m, 0, m->m_pkthdr.len,
 496                                     (void *)xr->buf.vaddr);
 497                         seg[0].ds_addr = (bus_addr_t)xr->buf.paddr;
 498                 } else
 499                         seg[0].ds_addr = mtod(m, bus_addr_t);
 500                 seg[0].ds_len = m->m_pkthdr.len;
 501                 break;
 502         case XR_TYPE_BIO:
 503         case XR_TYPE_VIRT:
 504         default:
 505                 panic("implement me\n");
 506         }
 507
 508         return (nsegs);
 509 }
 510
 511 static int
 512 xdma_load_data(xdma_channel_t *xchan,
 513     struct xdma_request *xr, struct bus_dma_segment *seg)
 514 {
 515         xdma_controller_t *xdma;
 516         int error;
 517         int nsegs;
 518
 519         xdma = xchan->xdma;
 520
 521         error = 0;
 522         nsegs = 0;
 523
 524         if (xchan->caps & XCHAN_CAP_BUSDMA)
 525                 nsegs = _xdma_load_data_busdma(xchan, xr, seg);
 526         else
 527                 nsegs = _xdma_load_data(xchan, xr, seg);
 528         if (nsegs == 0)
 529                 return (0); /* Try again later. */
 530
 531         xr->buf.nsegs = nsegs;
 532         xr->buf.nsegs_left = nsegs;
 533
 534         return (nsegs);
 535 }
 536
 537 static int
 538 xdma_process(xdma_channel_t *xchan,
 539     struct xdma_sglist *sg)
 540 {
 541         struct bus_dma_segment seg[XDMA_MAX_SEG];
 542         struct xdma_request *xr;
 543         struct xdma_request *xr_tmp;
 544         xdma_controller_t *xdma;
 545         uint32_t capacity;
 546         uint32_t n;
 547         uint32_t c;
 548         int nsegs;
 549         int ret;
 550
 551         XCHAN_ASSERT_LOCKED(xchan);
 552
 553         xdma = xchan->xdma;
 554
 555         n = 0;
 556         c = 0;
 557
 558         ret = XDMA_CHANNEL_CAPACITY(xdma->dma_dev, xchan, &capacity);
 559         if (ret != 0) {
 560                 device_printf(xdma->dev,
 561                     "%s: Can't get DMA controller capacity.\n", __func__);
 562                 return (-1);
 563         }
 564
 565         TAILQ_FOREACH_SAFE(xr, &xchan->queue_in, xr_next, xr_tmp) {
 566                 switch (xr->req_type) {
 567                 case XR_TYPE_MBUF:
 568                         if ((xchan->caps & XCHAN_CAP_NOSEG) ||
 569                             (c > xchan->maxnsegs))
 570                                 c = xdma_mbuf_defrag(xchan, xr);
 571                         break;
 572                 case XR_TYPE_BIO:
 573                 case XR_TYPE_VIRT:
 574                 default:
 575                         c = 1;
 576                 }
 577
 578                 if (capacity <= (c + n)) {
 579                         /*
 580                          * No space yet available for the entire
 581                          * request in the DMA engine.
 582                          */
 583                         break;
 584                 }
 585
 586                 if ((c + n + xchan->maxnsegs) >= XDMA_SGLIST_MAXLEN) {
 587                         /* Sglist is full. */
 588                         break;
 589                 }
 590
 591                 nsegs = xdma_load_data(xchan, xr, seg);
 592                 if (nsegs == 0)
 593                         break;
 594
 595                 xdma_sglist_add(&sg[n], seg, nsegs, xr);
 596                 n += nsegs;
 597
 598                 QUEUE_IN_LOCK(xchan);
 599                 TAILQ_REMOVE(&xchan->queue_in, xr, xr_next);
 600                 QUEUE_IN_UNLOCK(xchan);
 601
 602                 QUEUE_PROC_LOCK(xchan);
 603                 TAILQ_INSERT_TAIL(&xchan->processing, xr, xr_next);
 604                 QUEUE_PROC_UNLOCK(xchan);
 605         }
 606
 607         return (n);
 608 }
 609
 610 int
 611 xdma_queue_submit_sg(xdma_channel_t *xchan)
 612 {
 613         struct xdma_sglist *sg;
 614         xdma_controller_t *xdma;
 615         uint32_t sg_n;
 616         int ret;
 617
 618         xdma = xchan->xdma;
 619         KASSERT(xdma != NULL, ("xdma is NULL"));
 620
 621         XCHAN_ASSERT_LOCKED(xchan);
 622
 623         sg = xchan->sg;
 624
 625         if ((xchan->caps & XCHAN_CAP_NOBUFS) == 0 &&
 626            (xchan->flags & XCHAN_BUFS_ALLOCATED) == 0) {
 627                 device_printf(xdma->dev,
 628                     "%s: Can't submit a transfer: no bufs\n",
 629                     __func__);
 630                 return (-1);
 631         }
 632
 633         sg_n = xdma_process(xchan, sg);
 634         if (sg_n == 0)
 635                 return (0); /* Nothing to submit */
 636
 637         /* Now submit sglist to DMA engine driver. */
 638         ret = XDMA_CHANNEL_SUBMIT_SG(xdma->dma_dev, xchan, sg, sg_n);
 639         if (ret != 0) {
 640                 device_printf(xdma->dev,
 641                     "%s: Can't submit an sglist.\n", __func__);
 642                 return (-1);
 643         }
 644
 645         return (0);
 646 }