sys/ia64/ia64/support.S

   1 /*-
   2  * Copyright (c) 1998 Doug Rabson
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28 /*-
  29  * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
  30  * All rights reserved.
  31  *
  32  * Author: Chris G. Demetriou
  33  *
  34  * Permission to use, copy, modify and distribute this software and
  35  * its documentation is hereby granted, provided that both the copyright
  36  * notice and this permission notice appear in all copies of the
  37  * software, derivative works or modified versions, and any portions
  38  * thereof, and that both notices appear in supporting documentation.
  39  *
  40  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  41  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  42  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  43  *
  44  * Carnegie Mellon requests users of this software to return to
  45  *
  46  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  47  *  School of Computer Science
  48  *  Carnegie Mellon University
  49  *  Pittsburgh PA 15213-3890
  50  *
  51  * any improvements or extensions that they make and grant Carnegie the
  52  * rights to redistribute these changes.
  53  */
  54
  55 #include <machine/asm.h>
  56 #include <machine/ia64_cpu.h>
  57 #include <assym.s>
  58
  59         .text
  60
  61 /*
  62  * ia64_change_mode:    change mode to/from physical mode
  63  *
  64  * Arguments:
  65  *      r14     psr for desired mode
  66  *
  67  * Modifies:
  68  *      r15-r19 scratch
  69  *      ar.bsp  tranlated to new mode
  70  */
  71 ENTRY_NOPROFILE(ia64_change_mode, 0)
  72         rsm     psr.i | psr.ic
  73         mov     r19=ar.rsc              // save rsc while we change mode
  74         tbit.nz p6,p7=r14,17            // physical or virtual ?
  75         ;;
  76         mov     ar.rsc=0                // turn off RSE
  77 (p6)    mov     r15=7                   // RR base for virtual addresses
  78 (p7)    mov     r15=0                   // RR base for physical addresses
  79         ;;
  80         flushrs                         // no dirty registers please
  81         srlz.i
  82         ;;
  83         mov     r16=ar.bsp
  84         mov     r17=rp
  85         mov     r18=ar.rnat
  86         ;;
  87         dep     r16=r15,r16,61,3        // new address of ar.bsp
  88         dep     r17=r15,r17,61,3        // new address of rp
  89         dep     sp=r15,sp,61,3          // new address of sp
  90         ;;
  91         mov     ar.bspstore=r16
  92         mov     rp=r17
  93         ;;
  94 1:      mov     r16=ip
  95         mov     ar.rnat=r18
  96         mov     cr.ipsr=r14             // psr for new mode
  97         ;;
  98         add     r16=2f-1b,r16           // address to rfi to
  99         ;;
 100         dep     r16=r15,r16,61,3        // new mode address for rfi
 101         ;;
 102         mov     cr.iip=r16              // setup for rfi
 103         mov     cr.ifs=r0
 104         ;;
 105         rfi
 106
 107 2:      mov     ar.rsc=r19              // restore ar.rsc
 108         br.ret.sptk.few rp              // now in new mode
 109 END(ia64_change_mode)
 110
 111 /*
 112  * ia64_physical_mode:  change mode to physical mode
 113  *
 114  * Return:
 115  *      ret0    psr to restore
 116  *
 117  * Modifies:
 118  *      r15-r18 scratch
 119  *      ar.bsp  tranlated to physical mode
 120  *      psr.i   cleared
 121  */
 122 ENTRY(ia64_physical_mode, 0)
 123         mov     r14=psr
 124         mov     ret0=psr
 125         movl    r15=(IA64_PSR_I|IA64_PSR_IT|IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_DFL|IA64_PSR_DFH)
 126         movl    r16=IA64_PSR_BN
 127         ;;
 128         andcm   r14=r14,r15             // clear various xT bits
 129         ;;
 130         or      r14=r14,r16             // make sure BN=1
 131         or      ret0=ret0,r16           // make sure BN=1
 132
 133         br.cond.sptk.many ia64_change_mode
 134 END(ia64_physical_mode)
 135
 136 /*
 137  * ia64_call_efi_physical:      call an EFI procedure in physical mode
 138  *
 139  * Arguments:
 140  *      in0             Address of EFI procedure descriptor
 141  *      in1-in5         Arguments to EFI procedure
 142  *
 143  * Return:
 144  *      ret0-ret3       return values from EFI
 145  *
 146  */
 147 ENTRY(ia64_call_efi_physical, 6)
 148         .prologue
 149         .regstk 6,4,5,0
 150         .save   ar.pfs,loc0
 151         alloc   loc0=ar.pfs,6,4,5,0
 152         ;;
 153         .save   rp,loc1
 154         mov     loc1=rp
 155         ;;
 156         .body
 157         br.call.sptk.many rp=ia64_physical_mode
 158         ;;
 159         mov     loc2=r8                 // psr to restore mode
 160         mov     loc3=gp                 // save kernel gp
 161         ld8     r14=[in0],8             // function address
 162         ;;
 163         mov     out0=in1
 164         mov     out1=in2
 165         mov     out2=in3
 166         mov     out3=in4
 167         mov     out4=in5
 168         ld8     gp=[in0]                // function gp value
 169         ;;
 170         mov     b6=r14
 171         ;;
 172         br.call.sptk.many rp=b6         // call EFI procedure
 173         mov     gp=loc3                 // restore kernel gp
 174         ;;
 175         mov     r14=loc2                // psr to restore mode
 176         br.call.sptk.many rp=ia64_change_mode
 177         ;;
 178         mov     rp=loc1
 179         mov     ar.pfs=loc0
 180         ;;
 181         br.ret.sptk.many rp
 182 END(ia64_call_efi_physical)
 183
 184 /**************************************************************************/
 185
 186 ENTRY(fusufault, 0)
 187 {       .mib
 188         st8.rel         [r15]=r0                // Clear onfault.
 189         add             ret0=-1,r0
 190         br.ret.sptk     rp
 191         ;;
 192 }
 193 END(fusufault)
 194
 195 /*
 196  * casuword(u_long *p, u_long old, u_long new)
 197  *      Perform a compare-exchange in user space.
 198  */
 199 ENTRY(casuword, 3)
 200 {       .mlx
 201         add             r15=PC_CURTHREAD,r13
 202         movl            r14=VM_MAX_ADDRESS
 203         ;;
 204 }
 205 {       .mib
 206         ld8             r15=[r15]               // r15 = curthread
 207         cmp.geu         p6,p0=in0,r14
 208 (p6)    br.dpnt.few     1f
 209         ;;
 210 }
 211 {       .mlx
 212         add             r15=TD_PCB,r15
 213         movl            r14=fusufault
 214         ;;
 215 }
 216 {       .mmi
 217         ld8             r15=[r15]               // r15 = PCB
 218         ;;
 219         mov             ar.ccv=in1
 220         add             r15=PCB_ONFAULT,r15
 221         ;;
 222 }
 223 {       .mmi
 224         st8             [r15]=r14               // Set onfault
 225         ;;
 226         cmpxchg8.rel    ret0=[in0],in2,ar.ccv
 227         nop             0
 228         ;;
 229 }
 230 {       .mib
 231         st8.rel         [r15]=r0                // Clear onfault
 232         nop             0
 233         br.ret.sptk     rp
 234         ;;
 235 }
 236 1:
 237 {       .mib
 238         add             ret0=-1,r0
 239         nop             0
 240         br.ret.sptk     rp
 241         ;;
 242 }
 243 END(casuword)
 244
 245 /*
 246  * casuword32(uint32_t *p, uint32_t old, uint32_t new)
 247  *      Perform a 32-bit compare-exchange in user space.
 248  */
 249 ENTRY(casuword32, 3)
 250 {       .mlx
 251         add             r15=PC_CURTHREAD,r13
 252         movl            r14=VM_MAX_ADDRESS
 253         ;;
 254 }
 255 {       .mib
 256         ld8             r15=[r15]               // r15 = curthread
 257         cmp.geu         p6,p0=in0,r14
 258 (p6)    br.dpnt.few     1f
 259         ;;
 260 }
 261 {       .mlx
 262         add             r15=TD_PCB,r15
 263         movl            r14=fusufault
 264         ;;
 265 }
 266 {       .mmi
 267         ld8             r15=[r15]               // r15 = PCB
 268         ;;
 269         mov             ar.ccv=in1
 270         add             r15=PCB_ONFAULT,r15
 271         ;;
 272 }
 273 {       .mmi
 274         st8             [r15]=r14               // Set onfault
 275         ;;
 276         cmpxchg4.rel    ret0=[in0],in2,ar.ccv
 277         nop             0
 278         ;;
 279 }
 280 {       .mib
 281         st8.rel         [r15]=r0                // Clear onfault
 282         nop             0
 283         br.ret.sptk     rp
 284         ;;
 285 }
 286 1:
 287 {       .mib
 288         add             ret0=-1,r0
 289         nop             0
 290         br.ret.sptk     rp
 291         ;;
 292 }
 293 END(casuword32)
 294
 295 /*
 296  * subyte(void *addr, int byte)
 297  * suword16(void *addr, int word)
 298  * suword32(void *addr, int word)
 299  * suword64|suword(void *addr, long word)
 300  *      Store in user space
 301  */
 302
 303 ENTRY(subyte, 2)
 304 {       .mlx
 305         add             r15=PC_CURTHREAD,r13
 306         movl            r14=VM_MAX_ADDRESS
 307         ;;
 308 }
 309 {       .mib
 310         ld8             r15=[r15]               // r15 = curthread
 311         cmp.geu         p6,p0=in0,r14
 312 (p6)    br.dpnt.few     1f
 313         ;;
 314 }
 315 {       .mlx
 316         add             r15=TD_PCB,r15
 317         movl            r14=fusufault
 318         ;;
 319 }
 320 {       .mmi
 321         ld8             r15=[r15]               // r15 = PCB
 322         ;;
 323         nop             0
 324         add             r15=PCB_ONFAULT,r15
 325         ;;
 326 }
 327 {       .mmi
 328         st8             [r15]=r14               // Set onfault
 329         ;;
 330         st1.rel         [in0]=in1
 331         nop             0
 332         ;;
 333 }
 334 {       .mib
 335         st8.rel         [r15]=r0                // Clear onfault
 336         mov             ret0=r0
 337         br.ret.sptk     rp
 338         ;;
 339 }
 340 1:
 341 {       .mib
 342         add             ret0=-1,r0
 343         nop             0
 344         br.ret.sptk     rp
 345         ;;
 346 }
 347 END(subyte)
 348
 349 ENTRY(suword16, 2)
 350 {       .mlx
 351         add             r15=PC_CURTHREAD,r13
 352         movl            r14=VM_MAX_ADDRESS
 353         ;;
 354 }
 355 {       .mib
 356         ld8             r15=[r15]               // r15 = curthread
 357         cmp.geu         p6,p0=in0,r14
 358 (p6)    br.dpnt.few     1f
 359         ;;
 360 }
 361 {       .mlx
 362         add             r15=TD_PCB,r15
 363         movl            r14=fusufault
 364         ;;
 365 }
 366 {       .mmi
 367         ld8             r15=[r15]               // r15 = PCB
 368         ;;
 369         nop             0
 370         add             r15=PCB_ONFAULT,r15
 371         ;;
 372 }
 373 {       .mmi
 374         st8             [r15]=r14               // Set onfault
 375         ;;
 376         st2.rel         [in0]=in1
 377         nop             0
 378         ;;
 379 }
 380 {       .mib
 381         st8.rel         [r15]=r0                // Clear onfault
 382         mov             ret0=r0
 383         br.ret.sptk     rp
 384         ;;
 385 }
 386 1:
 387 {       .mib
 388         add             ret0=-1,r0
 389         nop             0
 390         br.ret.sptk     rp
 391         ;;
 392 }
 393 END(suword16)
 394
 395 ENTRY(suword32, 2)
 396 {       .mlx
 397         add             r15=PC_CURTHREAD,r13
 398         movl            r14=VM_MAX_ADDRESS
 399         ;;
 400 }
 401 {       .mib
 402         ld8             r15=[r15]               // r15 = curthread
 403         cmp.geu         p6,p0=in0,r14
 404 (p6)    br.dpnt.few     1f
 405         ;;
 406 }
 407 {       .mlx
 408         add             r15=TD_PCB,r15
 409         movl            r14=fusufault
 410         ;;
 411 }
 412 {       .mmi
 413         ld8             r15=[r15]               // r15 = PCB
 414         ;;
 415         nop             0
 416         add             r15=PCB_ONFAULT,r15
 417         ;;
 418 }
 419 {       .mmi
 420         st8             [r15]=r14               // Set onfault
 421         ;;
 422         st4.rel         [in0]=in1
 423         nop             0
 424         ;;
 425 }
 426 {       .mib
 427         st8.rel         [r15]=r0                // Clear onfault
 428         mov             ret0=r0
 429         br.ret.sptk     rp
 430         ;;
 431 }
 432 1:
 433 {       .mib
 434         add             ret0=-1,r0
 435         nop             0
 436         br.ret.sptk     rp
 437         ;;
 438 }
 439 END(suword32)
 440
 441 ENTRY(suword64, 2)
 442 XENTRY(suword)
 443 {       .mlx
 444         add             r15=PC_CURTHREAD,r13
 445         movl            r14=VM_MAX_ADDRESS
 446         ;;
 447 }
 448 {       .mib
 449         ld8             r15=[r15]               // r15 = curthread
 450         cmp.geu         p6,p0=in0,r14
 451 (p6)    br.dpnt.few     1f
 452         ;;
 453 }
 454 {       .mlx
 455         add             r15=TD_PCB,r15
 456         movl            r14=fusufault
 457         ;;
 458 }
 459 {       .mmi
 460         ld8             r15=[r15]               // r15 = PCB
 461         ;;
 462         nop             0
 463         add             r15=PCB_ONFAULT,r15
 464         ;;
 465 }
 466 {       .mmi
 467         st8             [r15]=r14               // Set onfault
 468         ;;
 469         st8.rel         [in0]=in1
 470         nop             0
 471         ;;
 472 }
 473 {       .mib
 474         st8.rel         [r15]=r0                // Clear onfault
 475         mov             ret0=r0
 476         br.ret.sptk     rp
 477         ;;
 478 }
 479 1:
 480 {       .mib
 481         add             ret0=-1,r0
 482         nop             0
 483         br.ret.sptk     rp
 484         ;;
 485 }
 486 END(suword64)
 487
 488 /*
 489  * fubyte(void *addr, int byte)
 490  * fuword16(void *addr, int word)
 491  * fuword32(void *addr, int word)
 492  * fuword64|fuword(void *addr, long word)
 493  *      Fetch from user space
 494  */
 495
 496 ENTRY(fubyte, 1)
 497 {       .mlx
 498         add             r15=PC_CURTHREAD,r13
 499         movl            r14=VM_MAX_ADDRESS
 500         ;;
 501 }
 502 {       .mib
 503         ld8             r15=[r15]               // r15 = curthread
 504         cmp.geu         p6,p0=in0,r14
 505 (p6)    br.dpnt.few     1f
 506         ;;
 507 }
 508 {       .mlx
 509         add             r15=TD_PCB,r15
 510         movl            r14=fusufault
 511         ;;
 512 }
 513 {       .mmi
 514         ld8             r15=[r15]               // r15 = PCB
 515         ;;
 516         nop             0
 517         add             r15=PCB_ONFAULT,r15
 518         ;;
 519 }
 520 {       .mmi
 521         st8             [r15]=r14               // Set onfault
 522         ;;
 523         mf
 524         nop             0
 525         ;;
 526 }
 527 {       .mmb
 528         ld1             ret0=[in0]
 529         st8.rel         [r15]=r0                // Clear onfault
 530         br.ret.sptk     rp
 531         ;;
 532 }
 533 1:
 534 {       .mib
 535         add             ret0=-1,r0
 536         nop             0
 537         br.ret.sptk     rp
 538         ;;
 539 }
 540 END(fubyte)
 541
 542 ENTRY(fuword16, 2)
 543 {       .mlx
 544         add             r15=PC_CURTHREAD,r13
 545         movl            r14=VM_MAX_ADDRESS
 546         ;;
 547 }
 548 {       .mib
 549         ld8             r15=[r15]               // r15 = curthread
 550         cmp.geu         p6,p0=in0,r14
 551 (p6)    br.dpnt.few     1f
 552         ;;
 553 }
 554 {       .mlx
 555         add             r15=TD_PCB,r15
 556         movl            r14=fusufault
 557         ;;
 558 }
 559 {       .mmi
 560         ld8             r15=[r15]               // r15 = PCB
 561         ;;
 562         nop             0
 563         add             r15=PCB_ONFAULT,r15
 564         ;;
 565 }
 566 {       .mmi
 567         st8             [r15]=r14               // Set onfault
 568         ;;
 569         mf
 570         nop             0
 571         ;;
 572 }
 573 {       .mmb
 574         ld2             ret0=[in0]
 575         st8.rel         [r15]=r0                // Clear onfault
 576         br.ret.sptk     rp
 577         ;;
 578 }
 579 1:
 580 {       .mib
 581         add             ret0=-1,r0
 582         nop             0
 583         br.ret.sptk     rp
 584         ;;
 585 }
 586 END(fuword16)
 587
 588 ENTRY(fuword32, 2)
 589 {       .mlx
 590         add             r15=PC_CURTHREAD,r13
 591         movl            r14=VM_MAX_ADDRESS
 592         ;;
 593 }
 594 {       .mib
 595         ld8             r15=[r15]               // r15 = curthread
 596         cmp.geu         p6,p0=in0,r14
 597 (p6)    br.dpnt.few     1f
 598         ;;
 599 }
 600 {       .mlx
 601         add             r15=TD_PCB,r15
 602         movl            r14=fusufault
 603         ;;
 604 }
 605 {       .mmi
 606         ld8             r15=[r15]               // r15 = PCB
 607         ;;
 608         nop             0
 609         add             r15=PCB_ONFAULT,r15
 610         ;;
 611 }
 612 {       .mmi
 613         st8             [r15]=r14               // Set onfault
 614         ;;
 615         mf
 616         nop             0
 617         ;;
 618 }
 619 {       .mmb
 620         ld4             ret0=[in0]
 621         st8.rel         [r15]=r0                // Clear onfault
 622         br.ret.sptk     rp
 623         ;;
 624 }
 625 1:
 626 {       .mib
 627         add             ret0=-1,r0
 628         nop             0
 629         br.ret.sptk     rp
 630         ;;
 631 }
 632 END(fuword32)
 633
 634 ENTRY(fuword64, 2)
 635 XENTRY(fuword)
 636 {       .mlx
 637         add             r15=PC_CURTHREAD,r13
 638         movl            r14=VM_MAX_ADDRESS
 639         ;;
 640 }
 641 {       .mib
 642         ld8             r15=[r15]               // r15 = curthread
 643         cmp.geu         p6,p0=in0,r14
 644 (p6)    br.dpnt.few     1f
 645         ;;
 646 }
 647 {       .mlx
 648         add             r15=TD_PCB,r15
 649         movl            r14=fusufault
 650         ;;
 651 }
 652 {       .mmi
 653         ld8             r15=[r15]               // r15 = PCB
 654         ;;
 655         nop             0
 656         add             r15=PCB_ONFAULT,r15
 657         ;;
 658 }
 659 {       .mmi
 660         st8             [r15]=r14               // Set onfault
 661         ;;
 662         mf
 663         nop             0
 664         ;;
 665 }
 666 {       .mmb
 667         ld8             ret0=[in0]
 668         st8.rel         [r15]=r0                // Clear onfault
 669         br.ret.sptk     rp
 670         ;;
 671 }
 672 1:
 673 {       .mib
 674         add             ret0=-1,r0
 675         nop             0
 676         br.ret.sptk     rp
 677         ;;
 678 }
 679 END(fuword64)
 680
 681 /*
 682  * fuswintr(void *addr)
 683  * suswintr(void *addr)
 684  */
 685
 686 ENTRY(fuswintr, 1)
 687 {       .mib
 688         add             ret0=-1,r0
 689         nop             0
 690         br.ret.sptk     rp
 691         ;;
 692 }
 693 END(fuswintr)
 694
 695 ENTRY(suswintr, 0)
 696 {       .mib
 697         add             ret0=-1,r0
 698         nop             0
 699         br.ret.sptk     rp
 700         ;;
 701 }
 702 END(suswintr)
 703
 704 /**************************************************************************/
 705
 706 /*
 707  * Copy a null-terminated string within the kernel's address space.
 708  * If lenp is not NULL, store the number of chars copied in *lenp
 709  *
 710  * int copystr(char *from, char *to, size_t len, size_t *lenp);
 711  */
 712 ENTRY(copystr, 4)
 713         mov     r14=in2                 // r14 = i = len
 714         cmp.eq  p6,p0=r0,in2
 715 (p6)    br.cond.spnt.few 2f             // if (len == 0), bail out
 716
 717 1:      ld1     r15=[in0],1             // read one byte
 718         ;;
 719         st1     [in1]=r15,1             // write that byte
 720         add     in2=-1,in2              // len--
 721         ;;
 722         cmp.eq  p6,p0=r0,r15
 723         cmp.ne  p7,p0=r0,in2
 724         ;;
 725 (p6)    br.cond.spnt.few 2f             // if (*from == 0), bail out
 726 (p7)    br.cond.sptk.few 1b             // if (len != 0) copy more
 727
 728 2:      cmp.eq  p6,p0=r0,in3
 729 (p6)    br.cond.dpnt.few 3f             // if (lenp != NULL)
 730         sub     r14=r14,in2             // *lenp = (i - len)
 731         ;;
 732         st8     [in3]=r14
 733
 734 3:      cmp.eq  p6,p0=r0,r15
 735 (p6)    br.cond.spnt.few 4f             // *from == '\0'; leave quietly
 736
 737         mov     ret0=ENAMETOOLONG       // *from != '\0'; error.
 738         br.ret.sptk.few rp
 739
 740 4:      mov     ret0=0                  // return 0.
 741         br.ret.sptk.few rp
 742 END(copystr)
 743
 744 ENTRY(copyinstr, 4)
 745         .prologue
 746         .regstk 4, 3, 4, 0
 747         .save   ar.pfs,loc0
 748         alloc   loc0=ar.pfs,4,3,4,0
 749         .save   rp,loc1
 750         mov     loc1=rp
 751         .body
 752
 753         movl    loc2=VM_MAX_ADDRESS             // make sure that src addr
 754         ;;
 755         cmp.geu p6,p0=in0,loc2                  // is in user space.
 756         ;;
 757 (p6)    br.cond.spnt.few copyerr                // if it's not, error out.
 758         movl    r14=copyerr                     // set up fault handler.
 759         add     r15=PC_CURTHREAD,r13            // find curthread
 760         ;;
 761         ld8     r15=[r15]
 762         ;;
 763         add     r15=TD_PCB,r15                  // find pcb
 764         ;;
 765         ld8     r15=[r15]
 766         ;;
 767         add     loc2=PCB_ONFAULT,r15
 768         ;;
 769         st8     [loc2]=r14
 770         ;;
 771         mov     out0=in0
 772         mov     out1=in1
 773         mov     out2=in2
 774         mov     out3=in3
 775         ;;
 776         br.call.sptk.few rp=copystr             // do the copy.
 777         st8     [loc2]=r0                       // kill the fault handler.
 778         mov     ar.pfs=loc0                     // restore ar.pfs
 779         mov     rp=loc1                         // restore ra.
 780         br.ret.sptk.few rp                      // ret0 left over from copystr
 781 END(copyinstr)
 782
 783 /*
 784  * Not the fastest bcopy in the world.
 785  */
 786 ENTRY(bcopy, 3)
 787         mov     ret0=r0                         // return zero for copy{in,out}
 788         ;;
 789         cmp.le  p6,p0=in2,r0                    // bail if len <= 0
 790 (p6)    br.ret.spnt.few rp
 791
 792         sub     r14=in1,in0 ;;                  // check for overlap
 793         cmp.ltu p6,p0=r14,in2                   // dst-src < len
 794 (p6)    br.cond.spnt.few 5f
 795
 796         extr.u  r14=in0,0,3                     // src & 7
 797         extr.u  r15=in1,0,3 ;;                  // dst & 7
 798         cmp.eq  p6,p0=r14,r15                   // different alignment?
 799 (p6)    br.cond.spnt.few 2f                     // branch if same alignment
 800
 801 1:      ld1     r14=[in0],1 ;;                  // copy bytewise
 802         st1     [in1]=r14,1
 803         add     in2=-1,in2 ;;                   // len--
 804         cmp.ne  p6,p0=r0,in2
 805 (p6)    br.cond.dptk.few 1b                     // loop
 806         br.ret.sptk.few rp                      // done
 807
 808 2:      cmp.eq  p6,p0=r14,r0                    // aligned?
 809 (p6)    br.cond.sptk.few 4f
 810
 811 3:      ld1     r14=[in0],1 ;;                  // copy bytewise
 812         st1     [in1]=r14,1
 813         extr.u  r15=in0,0,3                     // src & 7
 814         add     in2=-1,in2 ;;                   // len--
 815         cmp.eq  p6,p0=r0,in2                    // done?
 816         cmp.eq  p7,p0=r0,r15 ;;                 // aligned now?
 817 (p6)    br.ret.spnt.few rp                      // return if done
 818 (p7)    br.cond.spnt.few 4f                     // go to main copy
 819         br.cond.sptk.few 3b                     // more bytes to copy
 820
 821         // At this point, in2 is non-zero
 822
 823 4:      mov     r14=8 ;;
 824         cmp.ltu p6,p0=in2,r14 ;;                // len < 8?
 825 (p6)    br.cond.spnt.few 1b                     // byte copy the end
 826         ld8     r15=[in0],8 ;;                  // copy word
 827         st8     [in1]=r15,8
 828         add     in2=-8,in2 ;;                   // len -= 8
 829         cmp.ne  p6,p0=r0,in2                    // done?
 830 (p6)    br.cond.spnt.few 4b                     // again
 831
 832         br.ret.sptk.few rp                      // return
 833
 834         // Don't bother optimising overlap case
 835
 836 5:      add     in0=in0,in2
 837         add     in1=in1,in2 ;;
 838         add     in0=-1,in0
 839         add     in1=-1,in1 ;;
 840
 841 6:      ld1     r14=[in0],-1 ;;
 842         st1     [in1]=r14,-1
 843         add     in2=-1,in2 ;;
 844         cmp.ne  p6,p0=r0,in2
 845 (p6)    br.cond.spnt.few 6b
 846
 847         br.ret.sptk.few rp
 848 END(bcopy)
 849
 850 ENTRY(memcpy,3)
 851         mov     r14=in0 ;;
 852         mov     in0=in1 ;;
 853         mov     in1=r14
 854         br.cond.sptk.few bcopy
 855 END(memcpy)
 856
 857 ENTRY(copyin, 3)
 858         .prologue
 859         .regstk 3, 3, 3, 0
 860         .save   ar.pfs,loc0
 861         alloc   loc0=ar.pfs,3,3,3,0
 862         .save   rp,loc1
 863         mov     loc1=rp
 864         .body
 865
 866         movl    loc2=VM_MAX_ADDRESS             // make sure that src addr
 867         ;;
 868         cmp.geu p6,p0=in0,loc2                  // is in user space.
 869         ;;
 870 (p6)    br.cond.spnt.few copyerr                // if it's not, error out.
 871         movl    r14=copyerr                     // set up fault handler.
 872         add     r15=PC_CURTHREAD,r13            // find curthread
 873         ;;
 874         ld8     r15=[r15]
 875         ;;
 876         add     r15=TD_PCB,r15                  // find pcb
 877         ;;
 878         ld8     r15=[r15]
 879         ;;
 880         add     loc2=PCB_ONFAULT,r15
 881         ;;
 882         st8     [loc2]=r14
 883         ;;
 884         mov     out0=in0
 885         mov     out1=in1
 886         mov     out2=in2
 887         ;;
 888         br.call.sptk.few rp=bcopy               // do the copy.
 889         st8     [loc2]=r0                       // kill the fault handler.
 890         mov     ar.pfs=loc0                     // restore ar.pfs
 891         mov     rp=loc1                         // restore ra.
 892         br.ret.sptk.few rp                      // ret0 left over from bcopy
 893 END(copyin)
 894
 895 ENTRY(copyout, 3)
 896         .prologue
 897         .regstk 3, 3, 3, 0
 898         .save   ar.pfs,loc0
 899         alloc   loc0=ar.pfs,3,3,3,0
 900         .save   rp,loc1
 901         mov     loc1=rp
 902         .body
 903
 904         movl    loc2=VM_MAX_ADDRESS             // make sure that dest addr
 905         ;;
 906         cmp.geu p6,p0=in1,loc2                  // is in user space.
 907         ;;
 908 (p6)    br.cond.spnt.few copyerr                // if it's not, error out.
 909         movl    r14=copyerr                     // set up fault handler.
 910         add     r15=PC_CURTHREAD,r13            // find curthread
 911         ;;
 912         ld8     r15=[r15]
 913         ;;
 914         add     r15=TD_PCB,r15                  // find pcb
 915         ;;
 916         ld8     r15=[r15]
 917         ;;
 918         add     loc2=PCB_ONFAULT,r15
 919         ;;
 920         st8     [loc2]=r14
 921         ;;
 922         mov     out0=in0
 923         mov     out1=in1
 924         mov     out2=in2
 925         ;;
 926         br.call.sptk.few rp=bcopy               // do the copy.
 927         st8     [loc2]=r0                       // kill the fault handler.
 928         mov     ar.pfs=loc0                     // restore ar.pfs
 929         mov     rp=loc1                         // restore ra.
 930         br.ret.sptk.few rp                      // ret0 left over from bcopy
 931 END(copyout)
 932
 933 ENTRY(copyerr, 0)
 934         add     r14=PC_CURTHREAD,r13 ;;         // find curthread
 935         ld8     r14=[r14] ;;
 936         add     r14=TD_PCB,r14 ;;               // curthread->td_addr
 937         ld8     r14=[r14] ;;
 938         add     r14=PCB_ONFAULT,r14 ;;          // &curthread->td_pcb->pcb_onfault
 939         st8     [r14]=r0                        // reset fault handler
 940
 941         mov     ret0=EFAULT                     // return EFAULT
 942         br.ret.sptk.few rp
 943 END(copyerr)
 944
 945 #if defined(GPROF)
 946 /*
 947  * Important registers:
 948  *      r8      structure return address
 949  *      rp      our return address
 950  *      in0     caller's ar.pfs
 951  *      in1     caller's gp
 952  *      in2     caller's rp
 953  *      in3     GOT entry
 954  *      ar.pfs  our pfs
 955  */
 956 ENTRY_NOPROFILE(_mcount, 4)
 957         alloc           loc0 = ar.pfs, 4, 3, 2, 0
 958         mov             loc1 = r8
 959         mov             loc2 = rp
 960         ;;
 961         mov             out0 = in2
 962         mov             out1 = rp
 963         br.call.sptk    rp = __mcount
 964         ;;
 965 1:
 966         mov             gp = in1
 967         mov             r14 = ip
 968         mov             b7 = loc2
 969         ;;
 970         add             r14 = 2f - 1b, r14
 971         mov             ar.pfs = loc0
 972         mov             rp = in2
 973         ;;
 974         mov             b7 = r14
 975         mov             b6 = loc2
 976         mov             r8 = loc1
 977         mov             r14 = in0
 978         br.ret.sptk     b7
 979         ;;
 980 2:
 981         mov             ar.pfs = r14
 982         br.sptk         b6
 983         ;;
 984 END(_mcount)
 985 #endif