sys/dev/xdma/xdma_sg.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause
   3  *
   4  * Copyright (c) 2018-2019 Ruslan Bukin <br@bsdpad.com>
   5  *
   6  * This software was developed by SRI International and the University of
   7  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
   8  * ("CTSRD"), as part of the DARPA CRASH research programme.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  */
  31
  32 #include <sys/cdefs.h>
  33 __FBSDID("$FreeBSD$");
  34
  35 #include "opt_platform.h"
  36 #include <sys/param.h>
  37 #include <sys/conf.h>
  38 #include <sys/bus.h>
  39 #include <sys/kernel.h>
  40 #include <sys/lock.h>
  41 #include <sys/malloc.h>
  42 #include <sys/mbuf.h>
  43 #include <sys/mutex.h>
  44 #include <sys/rwlock.h>
  45
  46 #include <machine/bus.h>
  47
  48 #include <vm/vm.h>
  49 #include <vm/pmap.h>
  50 #include <vm/vm_extern.h>
  51 #include <vm/vm_page.h>
  52
  53 #ifdef FDT
  54 #include <dev/fdt/fdt_common.h>
  55 #include <dev/ofw/ofw_bus.h>
  56 #include <dev/ofw/ofw_bus_subr.h>
  57 #endif
  58
  59 #include <dev/xdma/xdma.h>
  60
  61 #include <xdma_if.h>
  62
  63 struct seg_load_request {
  64         struct bus_dma_segment *seg;
  65         uint32_t nsegs;
  66         uint32_t error;
  67 };
  68
  69 static void
  70 xchan_bufs_free_reserved(xdma_channel_t *xchan)
  71 {
  72         struct xdma_request *xr;
  73         vm_size_t size;
  74         int i;
  75
  76         for (i = 0; i < xchan->xr_num; i++) {
  77                 xr = &xchan->xr_mem[i];
  78                 size = xr->buf.size;
  79                 if (xr->buf.vaddr) {
  80                         pmap_kremove_device(xr->buf.vaddr, size);
  81                         kva_free(xr->buf.vaddr, size);
  82                         xr->buf.vaddr = 0;
  83                 }
  84                 if (xr->buf.paddr) {
  85                         vmem_free(xchan->vmem, xr->buf.paddr, size);
  86                         xr->buf.paddr = 0;
  87                 }
  88                 xr->buf.size = 0;
  89         }
  90 }
  91
  92 static int
  93 xchan_bufs_alloc_reserved(xdma_channel_t *xchan)
  94 {
  95         xdma_controller_t *xdma;
  96         struct xdma_request *xr;
  97         vmem_addr_t addr;
  98         vm_size_t size;
  99         int i;
 100
 101         xdma = xchan->xdma;
 102
 103         if (xchan->vmem == NULL)
 104                 return (ENOBUFS);
 105
 106         for (i = 0; i < xchan->xr_num; i++) {
 107                 xr = &xchan->xr_mem[i];
 108                 size = round_page(xchan->maxsegsize);
 109                 if (vmem_alloc(xchan->vmem, size,
 110                     M_BESTFIT | M_NOWAIT, &addr)) {
 111                         device_printf(xdma->dev,
 112                             "%s: Can't allocate memory\n", __func__);
 113                         xchan_bufs_free_reserved(xchan);
 114                         return (ENOMEM);
 115                 }
 116
 117                 xr->buf.size = size;
 118                 xr->buf.paddr = addr;
 119                 xr->buf.vaddr = kva_alloc(size);
 120                 if (xr->buf.vaddr == 0) {
 121                         device_printf(xdma->dev,
 122                             "%s: Can't allocate KVA\n", __func__);
 123                         xchan_bufs_free_reserved(xchan);
 124                         return (ENOMEM);
 125                 }
 126                 pmap_kenter_device(xr->buf.vaddr, size, addr);
 127         }
 128
 129         return (0);
 130 }
 131
 132 static int
 133 xchan_bufs_alloc_busdma(xdma_channel_t *xchan)
 134 {
 135         xdma_controller_t *xdma;
 136         struct xdma_request *xr;
 137         int err;
 138         int i;
 139
 140         xdma = xchan->xdma;
 141
 142         /* Create bus_dma tag */
 143         err = bus_dma_tag_create(
 144             bus_get_dma_tag(xdma->dev), /* Parent tag. */
 145             xchan->alignment,           /* alignment */
 146             xchan->boundary,            /* boundary */
 147             xchan->lowaddr,             /* lowaddr */
 148             xchan->highaddr,            /* highaddr */
 149             NULL, NULL,                 /* filter, filterarg */
 150             xchan->maxsegsize * xchan->maxnsegs, /* maxsize */
 151             xchan->maxnsegs,            /* nsegments */
 152             xchan->maxsegsize,          /* maxsegsize */
 153             0,                          /* flags */
 154             NULL, NULL,                 /* lockfunc, lockarg */
 155             &xchan->dma_tag_bufs);
 156         if (err != 0) {
 157                 device_printf(xdma->dev,
 158                     "%s: Can't create bus_dma tag.\n", __func__);
 159                 return (-1);
 160         }
 161
 162         for (i = 0; i < xchan->xr_num; i++) {
 163                 xr = &xchan->xr_mem[i];
 164                 err = bus_dmamap_create(xchan->dma_tag_bufs, 0,
 165                     &xr->buf.map);
 166                 if (err != 0) {
 167                         device_printf(xdma->dev,
 168                             "%s: Can't create buf DMA map.\n", __func__);
 169
 170                         /* Cleanup. */
 171                         bus_dma_tag_destroy(xchan->dma_tag_bufs);
 172
 173                         return (-1);
 174                 }
 175         }
 176
 177         return (0);
 178 }
 179
 180 static int
 181 xchan_bufs_alloc(xdma_channel_t *xchan)
 182 {
 183         xdma_controller_t *xdma;
 184         int ret;
 185
 186         xdma = xchan->xdma;
 187
 188         if (xdma == NULL) {
 189                 device_printf(xdma->dev,
 190                     "%s: Channel was not allocated properly.\n", __func__);
 191                 return (-1);
 192         }
 193
 194         if (xchan->caps & XCHAN_CAP_BUSDMA)
 195                 ret = xchan_bufs_alloc_busdma(xchan);
 196         else {
 197                 ret = xchan_bufs_alloc_reserved(xchan);
 198         }
 199         if (ret != 0) {
 200                 device_printf(xdma->dev,
 201                     "%s: Can't allocate bufs.\n", __func__);
 202                 return (-1);
 203         }
 204
 205         xchan->flags |= XCHAN_BUFS_ALLOCATED;
 206
 207         return (0);
 208 }
 209
 210 static int
 211 xchan_bufs_free(xdma_channel_t *xchan)
 212 {
 213         struct xdma_request *xr;
 214         struct xchan_buf *b;
 215         int i;
 216
 217         if ((xchan->flags & XCHAN_BUFS_ALLOCATED) == 0)
 218                 return (-1);
 219
 220         if (xchan->caps & XCHAN_CAP_BUSDMA) {
 221                 for (i = 0; i < xchan->xr_num; i++) {
 222                         xr = &xchan->xr_mem[i];
 223                         b = &xr->buf;
 224                         bus_dmamap_destroy(xchan->dma_tag_bufs, b->map);
 225                 }
 226                 bus_dma_tag_destroy(xchan->dma_tag_bufs);
 227         } else
 228                 xchan_bufs_free_reserved(xchan);
 229
 230         xchan->flags &= ~XCHAN_BUFS_ALLOCATED;
 231
 232         return (0);
 233 }
 234
 235 void
 236 xdma_channel_free_sg(xdma_channel_t *xchan)
 237 {
 238
 239         xchan_bufs_free(xchan);
 240         xchan_sglist_free(xchan);
 241         xchan_bank_free(xchan);
 242 }
 243
 244 /*
 245  * Prepare xchan for a scatter-gather transfer.
 246  * xr_num - xdma requests queue size,
 247  * maxsegsize - maximum allowed scatter-gather list element size in bytes
 248  */
 249 int
 250 xdma_prep_sg(xdma_channel_t *xchan, uint32_t xr_num,
 251     bus_size_t maxsegsize, bus_size_t maxnsegs,
 252     bus_size_t alignment, bus_addr_t boundary,
 253     bus_addr_t lowaddr, bus_addr_t highaddr)
 254 {
 255         xdma_controller_t *xdma;
 256         int ret;
 257
 258         xdma = xchan->xdma;
 259
 260         KASSERT(xdma != NULL, ("xdma is NULL"));
 261
 262         if (xchan->flags & XCHAN_CONFIGURED) {
 263                 device_printf(xdma->dev,
 264                     "%s: Channel is already configured.\n", __func__);
 265                 return (-1);
 266         }
 267
 268         xchan->xr_num = xr_num;
 269         xchan->maxsegsize = maxsegsize;
 270         xchan->maxnsegs = maxnsegs;
 271         xchan->alignment = alignment;
 272         xchan->boundary = boundary;
 273         xchan->lowaddr = lowaddr;
 274         xchan->highaddr = highaddr;
 275
 276         if (xchan->maxnsegs > XDMA_MAX_SEG) {
 277                 device_printf(xdma->dev, "%s: maxnsegs is too big\n",
 278                     __func__);
 279                 return (-1);
 280         }
 281
 282         xchan_bank_init(xchan);
 283
 284         /* Allocate sglist. */
 285         ret = xchan_sglist_alloc(xchan);
 286         if (ret != 0) {
 287                 device_printf(xdma->dev,
 288                     "%s: Can't allocate sglist.\n", __func__);
 289                 return (-1);
 290         }
 291
 292         /* Allocate buffers if required. */
 293         if (xchan->caps & (XCHAN_CAP_BUSDMA | XCHAN_CAP_BOUNCE)) {
 294                 ret = xchan_bufs_alloc(xchan);
 295                 if (ret != 0) {
 296                         device_printf(xdma->dev,
 297                             "%s: Can't allocate bufs.\n", __func__);
 298
 299                         /* Cleanup */
 300                         xchan_sglist_free(xchan);
 301                         xchan_bank_free(xchan);
 302
 303                         return (-1);
 304                 }
 305         }
 306
 307         xchan->flags |= (XCHAN_CONFIGURED | XCHAN_TYPE_SG);
 308
 309         XCHAN_LOCK(xchan);
 310         ret = XDMA_CHANNEL_PREP_SG(xdma->dma_dev, xchan);
 311         if (ret != 0) {
 312                 device_printf(xdma->dev,
 313                     "%s: Can't prepare SG transfer.\n", __func__);
 314                 XCHAN_UNLOCK(xchan);
 315
 316                 return (-1);
 317         }
 318         XCHAN_UNLOCK(xchan);
 319
 320         return (0);
 321 }
 322
 323 void
 324 xchan_seg_done(xdma_channel_t *xchan,
 325     struct xdma_transfer_status *st)
 326 {
 327         struct xdma_request *xr;
 328         xdma_controller_t *xdma;
 329         struct xchan_buf *b;
 330         bus_addr_t addr;
 331
 332         xdma = xchan->xdma;
 333
 334         xr = TAILQ_FIRST(&xchan->processing);
 335         if (xr == NULL)
 336                 panic("request not found\n");
 337
 338         b = &xr->buf;
 339
 340         atomic_subtract_int(&b->nsegs_left, 1);
 341
 342         if (b->nsegs_left == 0) {
 343                 if (xchan->caps & XCHAN_CAP_BUSDMA) {
 344                         if (xr->direction == XDMA_MEM_TO_DEV)
 345                                 bus_dmamap_sync(xchan->dma_tag_bufs, b->map,
 346                                     BUS_DMASYNC_POSTWRITE);
 347                         else
 348                                 bus_dmamap_sync(xchan->dma_tag_bufs, b->map,
 349                                     BUS_DMASYNC_POSTREAD);
 350                         bus_dmamap_unload(xchan->dma_tag_bufs, b->map);
 351                 } else if (xchan->caps & XCHAN_CAP_BOUNCE) {
 352                         if (xr->req_type == XR_TYPE_MBUF &&
 353                             xr->direction == XDMA_DEV_TO_MEM)
 354                                 m_copyback(xr->m, 0, st->transferred,
 355                                     (void *)xr->buf.vaddr);
 356                 } else if (xchan->caps & XCHAN_CAP_IOMMU) {
 357                         if (xr->direction == XDMA_MEM_TO_DEV)
 358                                 addr = xr->src_addr;
 359                         else
 360                                 addr = xr->dst_addr;
 361                         xdma_iommu_remove_entry(xchan, addr);
 362                 }
 363                 xr->status.error = st->error;
 364                 xr->status.transferred = st->transferred;
 365
 366                 QUEUE_PROC_LOCK(xchan);
 367                 TAILQ_REMOVE(&xchan->processing, xr, xr_next);
 368                 QUEUE_PROC_UNLOCK(xchan);
 369
 370                 QUEUE_OUT_LOCK(xchan);
 371                 TAILQ_INSERT_TAIL(&xchan->queue_out, xr, xr_next);
 372                 QUEUE_OUT_UNLOCK(xchan);
 373         }
 374 }
 375
 376 static void
 377 xdma_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 378 {
 379         struct seg_load_request *slr;
 380         struct bus_dma_segment *seg;
 381         int i;
 382
 383         slr = arg;
 384         seg = slr->seg;
 385
 386         if (error != 0) {
 387                 slr->error = error;
 388                 return;
 389         }
 390
 391         slr->nsegs = nsegs;
 392
 393         for (i = 0; i < nsegs; i++) {
 394                 seg[i].ds_addr = segs[i].ds_addr;
 395                 seg[i].ds_len = segs[i].ds_len;
 396         }
 397 }
 398
 399 static int
 400 _xdma_load_data_busdma(xdma_channel_t *xchan, struct xdma_request *xr,
 401     struct bus_dma_segment *seg)
 402 {
 403         xdma_controller_t *xdma;
 404         struct seg_load_request slr;
 405         uint32_t nsegs;
 406         void *addr;
 407         int error;
 408
 409         xdma = xchan->xdma;
 410
 411         error = 0;
 412         nsegs = 0;
 413
 414         switch (xr->req_type) {
 415         case XR_TYPE_MBUF:
 416                 error = bus_dmamap_load_mbuf_sg(xchan->dma_tag_bufs,
 417                     xr->buf.map, xr->m, seg, &nsegs, BUS_DMA_NOWAIT);
 418                 break;
 419         case XR_TYPE_BIO:
 420                 slr.nsegs = 0;
 421                 slr.error = 0;
 422                 slr.seg = seg;
 423                 error = bus_dmamap_load_bio(xchan->dma_tag_bufs,
 424                     xr->buf.map, xr->bp, xdma_dmamap_cb, &slr, BUS_DMA_NOWAIT);
 425                 if (slr.error != 0) {
 426                         device_printf(xdma->dma_dev,
 427                             "%s: bus_dmamap_load failed, err %d\n",
 428                             __func__, slr.error);
 429                         return (0);
 430                 }
 431                 nsegs = slr.nsegs;
 432                 break;
 433         case XR_TYPE_VIRT:
 434                 switch (xr->direction) {
 435                 case XDMA_MEM_TO_DEV:
 436                         addr = (void *)xr->src_addr;
 437                         break;
 438                 case XDMA_DEV_TO_MEM:
 439                         addr = (void *)xr->dst_addr;
 440                         break;
 441                 default:
 442                         device_printf(xdma->dma_dev,
 443                             "%s: Direction is not supported\n", __func__);
 444                         return (0);
 445                 }
 446                 slr.nsegs = 0;
 447                 slr.error = 0;
 448                 slr.seg = seg;
 449                 error = bus_dmamap_load(xchan->dma_tag_bufs, xr->buf.map,
 450                     addr, (xr->block_len * xr->block_num),
 451                     xdma_dmamap_cb, &slr, BUS_DMA_NOWAIT);
 452                 if (slr.error != 0) {
 453                         device_printf(xdma->dma_dev,
 454                             "%s: bus_dmamap_load failed, err %d\n",
 455                             __func__, slr.error);
 456                         return (0);
 457                 }
 458                 nsegs = slr.nsegs;
 459                 break;
 460         default:
 461                 break;
 462         }
 463
 464         if (error != 0) {
 465                 if (error == ENOMEM) {
 466                         /*
 467                          * Out of memory. Try again later.
 468                          * TODO: count errors.
 469                          */
 470                 } else
 471                         device_printf(xdma->dma_dev,
 472                             "%s: bus_dmamap_load failed with err %d\n",
 473                             __func__, error);
 474                 return (0);
 475         }
 476
 477         if (xr->direction == XDMA_MEM_TO_DEV)
 478                 bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map,
 479                     BUS_DMASYNC_PREWRITE);
 480         else
 481                 bus_dmamap_sync(xchan->dma_tag_bufs, xr->buf.map,
 482                     BUS_DMASYNC_PREREAD);
 483
 484         return (nsegs);
 485 }
 486
 487 static int
 488 _xdma_load_data(xdma_channel_t *xchan, struct xdma_request *xr,
 489     struct bus_dma_segment *seg)
 490 {
 491         xdma_controller_t *xdma;
 492         struct mbuf *m;
 493         uint32_t nsegs;
 494         vm_offset_t va, addr;
 495         bus_addr_t pa;
 496         vm_prot_t prot;
 497
 498         xdma = xchan->xdma;
 499
 500         m = xr->m;
 501
 502         KASSERT(xchan->caps & XCHAN_CAP_NOSEG,
 503             ("Handling segmented data is not implemented here."));
 504
 505         nsegs = 1;
 506
 507         switch (xr->req_type) {
 508         case XR_TYPE_MBUF:
 509                 if (xchan->caps & XCHAN_CAP_BOUNCE) {
 510                         if (xr->direction == XDMA_MEM_TO_DEV)
 511                                 m_copydata(m, 0, m->m_pkthdr.len,
 512                                     (void *)xr->buf.vaddr);
 513                         seg[0].ds_addr = (bus_addr_t)xr->buf.paddr;
 514                 } else if (xchan->caps & XCHAN_CAP_IOMMU) {
 515                         addr = mtod(m, bus_addr_t);
 516                         pa = vtophys(addr);
 517
 518                         if (xr->direction == XDMA_MEM_TO_DEV)
 519                                 prot = VM_PROT_READ;
 520                         else
 521                                 prot = VM_PROT_WRITE;
 522
 523                         xdma_iommu_add_entry(xchan, &va,
 524                             pa, m->m_pkthdr.len, prot);
 525
 526                         /*
 527                          * Save VA so we can unload data later
 528                          * after completion of this transfer.
 529                          */
 530                         if (xr->direction == XDMA_MEM_TO_DEV)
 531                                 xr->src_addr = va;
 532                         else
 533                                 xr->dst_addr = va;
 534                         seg[0].ds_addr = va;
 535                 } else
 536                         seg[0].ds_addr = mtod(m, bus_addr_t);
 537                 seg[0].ds_len = m->m_pkthdr.len;
 538                 break;
 539         case XR_TYPE_BIO:
 540         case XR_TYPE_VIRT:
 541         default:
 542                 panic("implement me\n");
 543         }
 544
 545         return (nsegs);
 546 }
 547
 548 static int
 549 xdma_load_data(xdma_channel_t *xchan,
 550     struct xdma_request *xr, struct bus_dma_segment *seg)
 551 {
 552         xdma_controller_t *xdma;
 553         int error;
 554         int nsegs;
 555
 556         xdma = xchan->xdma;
 557
 558         error = 0;
 559         nsegs = 0;
 560
 561         if (xchan->caps & XCHAN_CAP_BUSDMA)
 562                 nsegs = _xdma_load_data_busdma(xchan, xr, seg);
 563         else
 564                 nsegs = _xdma_load_data(xchan, xr, seg);
 565         if (nsegs == 0)
 566                 return (0); /* Try again later. */
 567
 568         xr->buf.nsegs = nsegs;
 569         xr->buf.nsegs_left = nsegs;
 570
 571         return (nsegs);
 572 }
 573
 574 static int
 575 xdma_process(xdma_channel_t *xchan,
 576     struct xdma_sglist *sg)
 577 {
 578         struct bus_dma_segment seg[XDMA_MAX_SEG];
 579         struct xdma_request *xr;
 580         struct xdma_request *xr_tmp;
 581         xdma_controller_t *xdma;
 582         uint32_t capacity;
 583         uint32_t n;
 584         uint32_t c;
 585         int nsegs;
 586         int ret;
 587
 588         XCHAN_ASSERT_LOCKED(xchan);
 589
 590         xdma = xchan->xdma;
 591
 592         n = 0;
 593         c = 0;
 594
 595         ret = XDMA_CHANNEL_CAPACITY(xdma->dma_dev, xchan, &capacity);
 596         if (ret != 0) {
 597                 device_printf(xdma->dev,
 598                     "%s: Can't get DMA controller capacity.\n", __func__);
 599                 return (-1);
 600         }
 601
 602         TAILQ_FOREACH_SAFE(xr, &xchan->queue_in, xr_next, xr_tmp) {
 603                 switch (xr->req_type) {
 604                 case XR_TYPE_MBUF:
 605                         if ((xchan->caps & XCHAN_CAP_NOSEG) ||
 606                             (c > xchan->maxnsegs))
 607                                 c = xdma_mbuf_defrag(xchan, xr);
 608                         break;
 609                 case XR_TYPE_BIO:
 610                 case XR_TYPE_VIRT:
 611                 default:
 612                         c = 1;
 613                 }
 614
 615                 if (capacity <= (c + n)) {
 616                         /*
 617                          * No space yet available for the entire
 618                          * request in the DMA engine.
 619                          */
 620                         break;
 621                 }
 622
 623                 if ((c + n + xchan->maxnsegs) >= XDMA_SGLIST_MAXLEN) {
 624                         /* Sglist is full. */
 625                         break;
 626                 }
 627
 628                 nsegs = xdma_load_data(xchan, xr, seg);
 629                 if (nsegs == 0)
 630                         break;
 631
 632                 xdma_sglist_add(&sg[n], seg, nsegs, xr);
 633                 n += nsegs;
 634
 635                 QUEUE_IN_LOCK(xchan);
 636                 TAILQ_REMOVE(&xchan->queue_in, xr, xr_next);
 637                 QUEUE_IN_UNLOCK(xchan);
 638
 639                 QUEUE_PROC_LOCK(xchan);
 640                 TAILQ_INSERT_TAIL(&xchan->processing, xr, xr_next);
 641                 QUEUE_PROC_UNLOCK(xchan);
 642         }
 643
 644         return (n);
 645 }
 646
 647 int
 648 xdma_queue_submit_sg(xdma_channel_t *xchan)
 649 {
 650         struct xdma_sglist *sg;
 651         xdma_controller_t *xdma;
 652         uint32_t sg_n;
 653         int ret;
 654
 655         xdma = xchan->xdma;
 656         KASSERT(xdma != NULL, ("xdma is NULL"));
 657
 658         XCHAN_ASSERT_LOCKED(xchan);
 659
 660         sg = xchan->sg;
 661
 662         if ((xchan->caps & (XCHAN_CAP_BOUNCE | XCHAN_CAP_BUSDMA)) &&
 663            (xchan->flags & XCHAN_BUFS_ALLOCATED) == 0) {
 664                 device_printf(xdma->dev,
 665                     "%s: Can't submit a transfer: no bufs\n",
 666                     __func__);
 667                 return (-1);
 668         }
 669
 670         sg_n = xdma_process(xchan, sg);
 671         if (sg_n == 0)
 672                 return (0); /* Nothing to submit */
 673
 674         /* Now submit sglist to DMA engine driver. */
 675         ret = XDMA_CHANNEL_SUBMIT_SG(xdma->dma_dev, xchan, sg, sg_n);
 676         if (ret != 0) {
 677                 device_printf(xdma->dev,
 678                     "%s: Can't submit an sglist.\n", __func__);
 679                 return (-1);
 680         }
 681
 682         return (0);
 683 }