| Palmer Dabbelt | 5d8544e | 2017-07-10 18:03:19 -0700 | [diff] [blame] | 1 | #include <linux/linkage.h> |
| 2 | #include <asm/asm.h> |
| 3 | #include <asm/csr.h> |
| 4 | |
| 5 | .altmacro |
| 6 | .macro fixup op reg addr lbl |
| 7 | LOCAL _epc |
| 8 | _epc: |
| 9 | \op \reg, \addr |
| 10 | .section __ex_table,"a" |
| 11 | .balign RISCV_SZPTR |
| 12 | RISCV_PTR _epc, \lbl |
| 13 | .previous |
| 14 | .endm |
| 15 | |
| 16 | ENTRY(__copy_user) |
| 17 | |
| 18 | /* Enable access to user memory */ |
| 19 | li t6, SR_SUM |
| 20 | csrs sstatus, t6 |
| 21 | |
| 22 | add a3, a1, a2 |
| 23 | /* Use word-oriented copy only if low-order bits match */ |
| 24 | andi t0, a0, SZREG-1 |
| 25 | andi t1, a1, SZREG-1 |
| 26 | bne t0, t1, 2f |
| 27 | |
| 28 | addi t0, a1, SZREG-1 |
| 29 | andi t1, a3, ~(SZREG-1) |
| 30 | andi t0, t0, ~(SZREG-1) |
| 31 | /* |
| 32 | * a3: terminal address of source region |
| 33 | * t0: lowest XLEN-aligned address in source |
| 34 | * t1: highest XLEN-aligned address in source |
| 35 | */ |
| 36 | bgeu t0, t1, 2f |
| 37 | bltu a1, t0, 4f |
| 38 | 1: |
| 39 | fixup REG_L, t2, (a1), 10f |
| 40 | fixup REG_S, t2, (a0), 10f |
| 41 | addi a1, a1, SZREG |
| 42 | addi a0, a0, SZREG |
| 43 | bltu a1, t1, 1b |
| 44 | 2: |
| 45 | bltu a1, a3, 5f |
| 46 | |
| 47 | 3: |
| 48 | /* Disable access to user memory */ |
| 49 | csrc sstatus, t6 |
| 50 | li a0, 0 |
| 51 | ret |
| 52 | 4: /* Edge case: unalignment */ |
| 53 | fixup lbu, t2, (a1), 10f |
| 54 | fixup sb, t2, (a0), 10f |
| 55 | addi a1, a1, 1 |
| 56 | addi a0, a0, 1 |
| 57 | bltu a1, t0, 4b |
| 58 | j 1b |
| 59 | 5: /* Edge case: remainder */ |
| 60 | fixup lbu, t2, (a1), 10f |
| 61 | fixup sb, t2, (a0), 10f |
| 62 | addi a1, a1, 1 |
| 63 | addi a0, a0, 1 |
| 64 | bltu a1, a3, 5b |
| 65 | j 3b |
| 66 | ENDPROC(__copy_user) |
| 67 | |
| 68 | |
| 69 | ENTRY(__clear_user) |
| 70 | |
| 71 | /* Enable access to user memory */ |
| 72 | li t6, SR_SUM |
| 73 | csrs sstatus, t6 |
| 74 | |
| 75 | add a3, a0, a1 |
| 76 | addi t0, a0, SZREG-1 |
| 77 | andi t1, a3, ~(SZREG-1) |
| 78 | andi t0, t0, ~(SZREG-1) |
| 79 | /* |
| 80 | * a3: terminal address of target region |
| 81 | * t0: lowest doubleword-aligned address in target region |
| 82 | * t1: highest doubleword-aligned address in target region |
| 83 | */ |
| 84 | bgeu t0, t1, 2f |
| 85 | bltu a0, t0, 4f |
| 86 | 1: |
| Alan Kao | ebcbd75 | 2018-05-08 10:59:33 +0800 | [diff] [blame^] | 87 | fixup REG_S, zero, (a0), 11f |
| Palmer Dabbelt | 5d8544e | 2017-07-10 18:03:19 -0700 | [diff] [blame] | 88 | addi a0, a0, SZREG |
| 89 | bltu a0, t1, 1b |
| 90 | 2: |
| 91 | bltu a0, a3, 5f |
| 92 | |
| 93 | 3: |
| 94 | /* Disable access to user memory */ |
| 95 | csrc sstatus, t6 |
| 96 | li a0, 0 |
| 97 | ret |
| 98 | 4: /* Edge case: unalignment */ |
| Alan Kao | ebcbd75 | 2018-05-08 10:59:33 +0800 | [diff] [blame^] | 99 | fixup sb, zero, (a0), 11f |
| Palmer Dabbelt | 5d8544e | 2017-07-10 18:03:19 -0700 | [diff] [blame] | 100 | addi a0, a0, 1 |
| 101 | bltu a0, t0, 4b |
| 102 | j 1b |
| 103 | 5: /* Edge case: remainder */ |
| Alan Kao | ebcbd75 | 2018-05-08 10:59:33 +0800 | [diff] [blame^] | 104 | fixup sb, zero, (a0), 11f |
| Palmer Dabbelt | 5d8544e | 2017-07-10 18:03:19 -0700 | [diff] [blame] | 105 | addi a0, a0, 1 |
| 106 | bltu a0, a3, 5b |
| 107 | j 3b |
| 108 | ENDPROC(__clear_user) |
| 109 | |
| 110 | .section .fixup,"ax" |
| 111 | .balign 4 |
| Alan Kao | ebcbd75 | 2018-05-08 10:59:33 +0800 | [diff] [blame^] | 112 | /* Fixup code for __copy_user(10) and __clear_user(11) */ |
| Palmer Dabbelt | 5d8544e | 2017-07-10 18:03:19 -0700 | [diff] [blame] | 113 | 10: |
| 114 | /* Disable access to user memory */ |
| 115 | csrs sstatus, t6 |
| Alan Kao | ebcbd75 | 2018-05-08 10:59:33 +0800 | [diff] [blame^] | 116 | mv a0, a2 |
| 117 | ret |
| 118 | 11: |
| 119 | csrs sstatus, t6 |
| 120 | mv a0, a1 |
| Palmer Dabbelt | 5d8544e | 2017-07-10 18:03:19 -0700 | [diff] [blame] | 121 | ret |
| 122 | .previous |