2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 * Assembly variants of various functions, for those that don't need generic C
32 * implementations. Currently this includes:
34 * - Direct-access versions of copyin/copyout methods.
35 * - These are used by Radix AIM pmap (ISA 3.0), and Book-E, to avoid
36 * unnecessary pmap_map_usr_ptr() calls.
40 #include "opt_sched.h"
42 #include <sys/syscall.h>
43 #include <sys/errno.h>
45 #include <machine/param.h>
46 #include <machine/asm.h>
47 #include <machine/spr.h>
48 #include <machine/trap.h>
49 #include <machine/vmparam.h>
76 #define ENTRY_DIRECT(x) ENTRY(x ## _direct)
78 #define ENTRY_DIRECT(x) ENTRY(x)
92 #define VALIDATE_TRUNCATE_ADDR_COPY VALIDATE_ADDR_COPY
93 #define VALIDATE_ADDR_COPY(raddr, len) \
94 srdi %r0, raddr, 52 ;\
99 #define VALIDATE_ADDR_FUSU(raddr) ;\
100 srdi %r0, raddr, 52 ;\
116 /* %r0 is temporary */
118 * Validate address and length are valid.
119 * For VALIDATE_ADDR_COPY() have to account for wraparound.
121 #define VALIDATE_ADDR_COPY(raddr, len) \
122 lis %r0, VM_MAXUSER_ADDRESS@h ;\
123 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\
126 add %r0, raddr, len ;\
127 cmplw 7, %r0, raddr ;\
128 blt- 7, copy_fault ;\
133 #define VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) \
134 lis %r0, VM_MAXUSER_ADDRESS@h ;\
135 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\
138 sub %r0, %r0, raddr ;\
140 isel len, len, %r0, 0 ;\
142 #define VALIDATE_ADDR_FUSU(raddr) \
143 lis %r0, VM_MAXUSER_ADDRESS@h ;\
144 ori %r0, %r0, VM_MAXUSER_ADDRESS@l ;\
150 #define PCPU(reg) mfsprg reg, 0
152 #define SET_COPYFAULT(raddr, rpcb, len) \
153 VALIDATE_ADDR_COPY(raddr, len) ;\
156 LOAD rpcb, PC_CURPCB(%r9) ;\
157 STORE %r0, PCB_ONFAULT(rpcb) ;\
159 #define SET_COPYFAULT_TRUNCATE(raddr, rpcb, len)\
160 VALIDATE_TRUNCATE_ADDR_COPY(raddr, len) ;\
163 LOAD rpcb, PC_CURPCB(%r9) ;\
164 STORE %r0, PCB_ONFAULT(rpcb)
166 #define SET_FUSUFAULT(raddr, rpcb) \
167 VALIDATE_ADDR_FUSU(raddr) ;\
170 LOAD rpcb, PC_CURPCB(%r9) ;\
171 STORE %r0, PCB_ONFAULT(rpcb)
173 #define CLEAR_FAULT_NO_CLOBBER(rpcb) \
175 LOAD rpcb, PC_CURPCB(%r9) ;\
177 STORE %r0, PCB_ONFAULT(rpcb)
179 #define CLEAR_FAULT(rpcb) \
180 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\
184 * bcopy(src, dst, len)
187 * %r7 is the pcb pointer
189 * %r0 and %r8-%r10 are volatile
190 * %r11 and %r12 are generally volatile, used in linking and exception
191 * handling. Can be clobbered here.
193 * Does not allocate or use stack space, but clobbers all volatile registers.
209 #define Thresh WORD * 8
213 #define WORDS(n) (32 - LOG_WORD - W##n)
225 .Lsmall: /* < 8 words remaining */
239 0: /* < 4 words remaining */
247 1: /* < 2 words remaining */
253 2: /* < 1 word remaining */
260 3: /* < 4 bytes remaining */
267 4: /* < 2 bytes remaining */
278 andi. t6, t3, WORD-1 /* Align rd to word size */
290 srwi. t2, rl, LOOP_LOG /* length >> log_2(loop_size) => 8W iterations */
325 * copyout(from_kernel, to_user, len)
328 ENTRY_DIRECT(copyout)
330 SET_COPYFAULT(%r4, %r7, %r5)
337 * copyin(from_user, to_kernel, len)
342 SET_COPYFAULT(%r3, %r7, %r5)
348 * copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
352 ENTRY_DIRECT(copyinstr)
354 SET_COPYFAULT_TRUNCATE(%r3, %r7, %r5)
366 // NULL byte reached ?
373 /* skip storing length if done is NULL */
380 CLEAR_FAULT_NO_CLOBBER(%r7)
385 SET_FUSUFAULT(%r3, %r7)
390 #ifndef __powerpc64__
393 SET_FUSUFAULT(%r3, %r7)
399 ENTRY_DIRECT(suword32)
401 SET_FUSUFAULT(%r3, %r7)
407 ENTRY_DIRECT(suword64)
409 SET_FUSUFAULT(%r3, %r7)
415 SET_FUSUFAULT(%r3, %r7)
423 SET_FUSUFAULT(%r3, %r7)
425 CLEAR_FAULT_NO_CLOBBER(%r7)
428 ENTRY_DIRECT(fuword16)
430 SET_FUSUFAULT(%r3, %r7)
432 CLEAR_FAULT_NO_CLOBBER(%r7)
435 #ifndef __powerpc64__
436 ENTRY_DIRECT(fueword)
438 SET_FUSUFAULT(%r3, %r7)
444 ENTRY_DIRECT(fueword32)
446 SET_FUSUFAULT(%r3, %r7)
453 ENTRY_DIRECT(fueword)
455 SET_FUSUFAULT(%r3, %r7)
461 ENTRY_DIRECT(fueword64)
463 SET_FUSUFAULT(%r3, %r7)
471 * casueword(volatile u_long *base, u_long old, u_long *oldp, u_long new)
475 #define CASUEWORD32(raddr, rpcb) ;\
477 SET_FUSUFAULT(raddr, rpcb) ;\
483 stwcx. %r6, 0, %r3 ;\
487 stwcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\
492 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\
496 ENTRY_DIRECT(casueword32)
497 CASUEWORD32(%r3, %r7)
500 #define CASUEWORD64(raddr, rpcb) ;\
502 SET_FUSUFAULT(raddr, rpcb) ;\
508 stdcx. %r6, 0, %r3 ;\
512 stdcx. %r0, 0, %r3 /* clear reservation (74xx) */ ;\
517 CLEAR_FAULT_NO_CLOBBER(rpcb) ;\
521 ENTRY_DIRECT(casueword)
522 CASUEWORD64(%r3, %r7)
524 ENTRY_DIRECT(casueword64)
525 CASUEWORD64(%r3, %r7)
527 ENTRY_DIRECT(casueword)
528 CASUEWORD32(%r3, %r7)
531 _NAKED_ENTRY(fusufault)
532 CLEAR_FAULT_NO_CLOBBER(%r7)
536 _NAKED_ENTRY(copy_fault)
537 CLEAR_FAULT_NO_CLOBBER(%r7)