2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2023 Chelsio Communications, Inc.
5 * Written by: John Baldwin <jhb@FreeBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/param.h>
31 #include <sys/memdesc.h>
32 #include <sys/systm.h>
36 #include <vm/vm_param.h>
37 #include <machine/bus.h>
40 phys_copyback(vm_paddr_t pa, int off, int size, const void *src)
47 KASSERT(PMAP_HAS_DMAP, ("direct-map required"));
51 page_off = pa & PAGE_MASK;
53 todo = min(PAGE_SIZE - page_off, size);
54 p = (void *)PHYS_TO_DMAP(pa);
64 vlist_copyback(struct bus_dma_segment *vlist, int sglist_cnt, int off,
65 int size, const void *src)
70 while (vlist->ds_len <= off) {
71 KASSERT(sglist_cnt > 1, ("out of sglist entries"));
80 KASSERT(sglist_cnt >= 1, ("out of sglist entries"));
83 if (todo > vlist->ds_len - off)
84 todo = vlist->ds_len - off;
86 memcpy((char *)(uintptr_t)vlist->ds_addr + off, p, todo);
96 plist_copyback(struct bus_dma_segment *plist, int sglist_cnt, int off,
97 int size, const void *src)
102 while (plist->ds_len <= off) {
103 KASSERT(sglist_cnt > 1, ("out of sglist entries"));
105 off -= plist->ds_len;
112 KASSERT(sglist_cnt >= 1, ("out of sglist entries"));
115 if (todo > plist->ds_len - off)
116 todo = plist->ds_len - off;
118 phys_copyback(plist->ds_addr, off, todo, p);
128 vmpages_copyback(vm_page_t *m, int off, int size, const void *src)
132 int error __diagused;
134 iov[0].iov_base = __DECONST(void *, src);
135 iov[0].iov_len = size;
139 uio.uio_resid = size;
140 uio.uio_segflg = UIO_SYSSPACE;
141 uio.uio_rw = UIO_WRITE;
142 error = uiomove_fromphys(m, off, size, &uio);
143 KASSERT(error == 0 && uio.uio_resid == 0, ("copy failed"));
147 memdesc_copyback(struct memdesc *mem, int off, int size, const void *src)
149 KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off));
150 KASSERT(size >= 0, ("%s: invalid size %d", __func__, off));
152 switch (mem->md_type) {
154 KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
155 memcpy((char *)mem->u.md_vaddr + off, src, size);
158 KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
159 phys_copyback(mem->u.md_paddr, off, size, src);
162 vlist_copyback(mem->u.md_list, mem->md_nseg, off, size, src);
165 plist_copyback(mem->u.md_list, mem->md_nseg, off, size, src);
168 panic("Use uiomove instead");
171 m_copyback(mem->u.md_mbuf, off, size, src);
173 case MEMDESC_VMPAGES:
174 KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
175 vmpages_copyback(mem->u.md_ma, mem->md_offset + off, size,
179 __assert_unreachable();
184 phys_copydata(vm_paddr_t pa, int off, int size, void *dst)
191 KASSERT(PMAP_HAS_DMAP, ("direct-map required"));
195 page_off = pa & PAGE_MASK;
197 todo = min(PAGE_SIZE - page_off, size);
198 p = (const void *)PHYS_TO_DMAP(pa);
208 vlist_copydata(struct bus_dma_segment *vlist, int sglist_cnt, int off,
214 while (vlist->ds_len <= off) {
215 KASSERT(sglist_cnt > 1, ("out of sglist entries"));
217 off -= vlist->ds_len;
224 KASSERT(sglist_cnt >= 1, ("out of sglist entries"));
227 if (todo > vlist->ds_len - off)
228 todo = vlist->ds_len - off;
230 memcpy(p, (char *)(uintptr_t)vlist->ds_addr + off, todo);
240 plist_copydata(struct bus_dma_segment *plist, int sglist_cnt, int off,
246 while (plist->ds_len <= off) {
247 KASSERT(sglist_cnt > 1, ("out of sglist entries"));
249 off -= plist->ds_len;
256 KASSERT(sglist_cnt >= 1, ("out of sglist entries"));
259 if (todo > plist->ds_len - off)
260 todo = plist->ds_len - off;
262 phys_copydata(plist->ds_addr, off, todo, p);
272 vmpages_copydata(vm_page_t *m, int off, int size, void *dst)
276 int error __diagused;
278 iov[0].iov_base = dst;
279 iov[0].iov_len = size;
283 uio.uio_resid = size;
284 uio.uio_segflg = UIO_SYSSPACE;
285 uio.uio_rw = UIO_READ;
286 error = uiomove_fromphys(m, off, size, &uio);
287 KASSERT(error == 0 && uio.uio_resid == 0, ("copy failed"));
291 memdesc_copydata(struct memdesc *mem, int off, int size, void *dst)
293 KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off));
294 KASSERT(size >= 0, ("%s: invalid size %d", __func__, off));
296 switch (mem->md_type) {
298 KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
299 memcpy(dst, (const char *)mem->u.md_vaddr + off, size);
302 KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
303 phys_copydata(mem->u.md_paddr, off, size, dst);
306 vlist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst);
309 plist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst);
312 panic("Use uiomove instead");
315 m_copydata(mem->u.md_mbuf, off, size, dst);
317 case MEMDESC_VMPAGES:
318 KASSERT(off + size <= mem->md_len, ("copy out of bounds"));
319 vmpages_copydata(mem->u.md_ma, mem->md_offset + off, size,
323 __assert_unreachable();