Gitiles
Code Review Sign In
review.shift-gmbh.com / SHIFTPHONES / mainline / linux / 18a1d5e1945385d9b5adc3fe11427ce4a9d2826e / . / arch / powerpc / lib / string_64.S
blob: 169872bc08928aa5fc566355ac0e5165985a1d00 [file] [log] [blame]
Thomas Gleixner1a59d1b82019-05-27 08:55:05 +0200[diff] [blame]1/* SPDX-License-Identifier: GPL-2.0-or-later */
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]2/*
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]3 *
4 * Copyright (C) IBM Corporation, 2012
5 *
6 * Author: Anton Blanchard <anton@au.ibm.com>
7 */
8
9#include <asm/ppc_asm.h>
Nicholas Piggin24bfa6a2016-10-13 16:42:53 +1100[diff] [blame]10#include <asm/linkage.h>
Anton Blanchardcf8fb552012-06-04 16:02:22 +0000[diff] [blame]11#include <asm/asm-offsets.h>
Al Viro9445aa12016-01-13 23:33:46 -0500[diff] [blame]12#include <asm/export.h>
Anton Blanchardcf8fb552012-06-04 16:02:22 +0000[diff] [blame]13
14 .section ".toc","aw"
15PPC64_CACHES:
16 .tc ppc64_caches[TC],ppc64_caches
17 .section ".text"
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]18
19/**
Andrew Donnellan61e3acd2019-12-10 00:22:21 +1100[diff] [blame]20 * __arch_clear_user: - Zero a block of memory in user space, with less checking.
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]21 * @to: Destination address, in user space.
22 * @n: Number of bytes to zero.
23 *
24 * Zero a block of memory in user space. Caller must check
25 * the specified block with access_ok() before calling this function.
26 *
27 * Returns number of bytes that could not be cleared.
28 * On success, this will be zero.
29 */
30
31 .macro err1
32100:
Nicholas Piggin24bfa6a2016-10-13 16:42:53 +1100[diff] [blame]33 EX_TABLE(100b,.Ldo_err1)
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]34 .endm
35
36 .macro err2
37200:
Nicholas Piggin24bfa6a2016-10-13 16:42:53 +1100[diff] [blame]38 EX_TABLE(200b,.Ldo_err2)
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]39 .endm
40
41 .macro err3
42300:
Nicholas Piggin24bfa6a2016-10-13 16:42:53 +1100[diff] [blame]43 EX_TABLE(300b,.Ldo_err3)
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]44 .endm
45
46.Ldo_err1:
47 mr r3,r8
48
49.Ldo_err2:
50 mtctr r4
511:
52err3; stb r0,0(r3)
53 addi r3,r3,1
54 addi r4,r4,-1
55 bdnz 1b
56
57.Ldo_err3:
58 mr r3,r4
59 blr
60
Andrew Donnellan61e3acd2019-12-10 00:22:21 +1100[diff] [blame]61_GLOBAL_TOC(__arch_clear_user)
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]62 cmpdi r4,32
63 neg r6,r3
64 li r0,0
65 blt .Lshort_clear
66 mr r8,r3
67 mtocrf 0x01,r6
68 clrldi r6,r6,(64-3)
69
70 /* Get the destination 8 byte aligned */
71 bf cr7*4+3,1f
72err1; stb r0,0(r3)
73 addi r3,r3,1
74
751: bf cr7*4+2,2f
76err1; sth r0,0(r3)
77 addi r3,r3,2
78
792: bf cr7*4+1,3f
80err1; stw r0,0(r3)
81 addi r3,r3,4
82
833: sub r4,r4,r6
Anton Blanchardcf8fb552012-06-04 16:02:22 +0000[diff] [blame]84
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]85 cmpdi r4,32
Anton Blanchardcf8fb552012-06-04 16:02:22 +0000[diff] [blame]86 cmpdi cr1,r4,512
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]87 blt .Lshort_clear
Anton Blanchardcf8fb552012-06-04 16:02:22 +0000[diff] [blame]88 bgt cr1,.Llong_clear
89
90.Lmedium_clear:
91 srdi r6,r4,5
Anton Blanchard17968fb2012-05-27 19:54:03 +0000[diff] [blame]92 mtctr r6
93
94 /* Do 32 byte chunks */
954:
96err2; std r0,0(r3)
97err2; std r0,8(r3)
98err2; std r0,16(r3)
99err2; std r0,24(r3)
100 addi r3,r3,32
101 addi r4,r4,-32
102 bdnz 4b
103
104.Lshort_clear:
105 /* up to 31 bytes to go */
106 cmpdi r4,16
107 blt 6f
108err2; std r0,0(r3)
109err2; std r0,8(r3)
110 addi r3,r3,16
111 addi r4,r4,-16
112
113 /* Up to 15 bytes to go */
1146: mr r8,r3
115 clrldi r4,r4,(64-4)
116 mtocrf 0x01,r4
117 bf cr7*4+0,7f
118err1; std r0,0(r3)
119 addi r3,r3,8
120
1217: bf cr7*4+1,8f
122err1; stw r0,0(r3)
123 addi r3,r3,4
124
1258: bf cr7*4+2,9f
126err1; sth r0,0(r3)
127 addi r3,r3,2
128
1299: bf cr7*4+3,10f
130err1; stb r0,0(r3)
131
13210: li r3,0
133 blr
Anton Blanchardcf8fb552012-06-04 16:02:22 +0000[diff] [blame]134
135.Llong_clear:
136 ld r5,PPC64_CACHES@toc(r2)
137
138 bf cr7*4+0,11f
139err2; std r0,0(r3)
140 addi r3,r3,8
141 addi r4,r4,-8
142
Benjamin Herrenschmidtbd067f82017-01-08 17:31:45 -0600[diff] [blame]143 /* Destination is 16 byte aligned, need to get it cache block aligned */
14411: lwz r7,DCACHEL1LOGBLOCKSIZE(r5)
145 lwz r9,DCACHEL1BLOCKSIZE(r5)
Anton Blanchardcf8fb552012-06-04 16:02:22 +0000[diff] [blame]146
147 /*
148 * With worst case alignment the long clear loop takes a minimum
149 * of 1 byte less than 2 cachelines.
150 */
151 sldi r10,r9,2
152 cmpd r4,r10
153 blt .Lmedium_clear
154
155 neg r6,r3
156 addi r10,r9,-1
157 and. r5,r6,r10
158 beq 13f
159
160 srdi r6,r5,4
161 mtctr r6
162 mr r8,r3
16312:
164err1; std r0,0(r3)
165err1; std r0,8(r3)
166 addi r3,r3,16
167 bdnz 12b
168
169 sub r4,r4,r5
170
17113: srd r6,r4,r7
172 mtctr r6
173 mr r8,r3
17414:
Andreas Schwab8a583c02017-08-05 19:55:11 +0200[diff] [blame]175err1; dcbz 0,r3
Anton Blanchardcf8fb552012-06-04 16:02:22 +0000[diff] [blame]176 add r3,r3,r9
177 bdnz 14b
178
179 and r4,r4,r10
180
181 cmpdi r4,32
182 blt .Lshort_clear
183 b .Lmedium_clear
Andrew Donnellan61e3acd2019-12-10 00:22:21 +1100[diff] [blame]184EXPORT_SYMBOL(__arch_clear_user)