GNU Linux-libre 4.14.266-gnu1
[releases.git] / arch / powerpc / lib / memcpy_64.S
1 /*
2  * Copyright (C) 2002 Paul Mackerras, IBM Corp.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version
7  * 2 of the License, or (at your option) any later version.
8  */
9 #include <asm/processor.h>
10 #include <asm/ppc_asm.h>
11 #include <asm/export.h>
12
13         .align  7
14 _GLOBAL_TOC(memcpy)
15 BEGIN_FTR_SECTION
16 #ifdef __LITTLE_ENDIAN__
17         cmpdi   cr7,r5,0
18 #else
19         std     r3,-STACKFRAMESIZE+STK_REG(R31)(r1)     /* save destination pointer for return value */
20 #endif
21 FTR_SECTION_ELSE
22 #ifndef SELFTEST
23         b       memcpy_power7
24 #endif
25 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY)
26 #ifdef __LITTLE_ENDIAN__
27         /* dumb little-endian memcpy that will get replaced at runtime */
28         addi r9,r3,-1
29         addi r4,r4,-1
30         beqlr cr7
31         mtctr r5
32 1:      lbzu r10,1(r4)
33         stbu r10,1(r9)
34         bdnz 1b
35         blr
36 #else
37         PPC_MTOCRF(0x01,r5)
38         cmpldi  cr1,r5,16
39         neg     r6,r3           # LS 3 bits = # bytes to 8-byte dest bdry
40         andi.   r6,r6,7
41         dcbt    0,r4
42         blt     cr1,.Lshort_copy
43 /* Below we want to nop out the bne if we're on a CPU that has the
44    CPU_FTR_UNALIGNED_LD_STD bit set and the CPU_FTR_CP_USE_DCBTZ bit
45    cleared.
46    At the time of writing the only CPU that has this combination of bits
47    set is Power6. */
48 BEGIN_FTR_SECTION
49         nop
50 FTR_SECTION_ELSE
51         bne     .Ldst_unaligned
52 ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \
53                     CPU_FTR_UNALIGNED_LD_STD)
54 .Ldst_aligned:
55         addi    r3,r3,-16
56 BEGIN_FTR_SECTION
57         andi.   r0,r4,7
58         bne     .Lsrc_unaligned
59 END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
60         srdi    r7,r5,4
61         ld      r9,0(r4)
62         addi    r4,r4,-8
63         mtctr   r7
64         andi.   r5,r5,7
65         bf      cr7*4+0,2f
66         addi    r3,r3,8
67         addi    r4,r4,8
68         mr      r8,r9
69         blt     cr1,3f
70 1:      ld      r9,8(r4)
71         std     r8,8(r3)
72 2:      ldu     r8,16(r4)
73         stdu    r9,16(r3)
74         bdnz    1b
75 3:      std     r8,8(r3)
76         beq     3f
77         addi    r3,r3,16
78 .Ldo_tail:
79         bf      cr7*4+1,1f
80         lwz     r9,8(r4)
81         addi    r4,r4,4
82         stw     r9,0(r3)
83         addi    r3,r3,4
84 1:      bf      cr7*4+2,2f
85         lhz     r9,8(r4)
86         addi    r4,r4,2
87         sth     r9,0(r3)
88         addi    r3,r3,2
89 2:      bf      cr7*4+3,3f
90         lbz     r9,8(r4)
91         stb     r9,0(r3)
92 3:      ld      r3,-STACKFRAMESIZE+STK_REG(R31)(r1)     /* return dest pointer */
93         blr
94
95 .Lsrc_unaligned:
96         srdi    r6,r5,3
97         addi    r5,r5,-16
98         subf    r4,r0,r4
99         srdi    r7,r5,4
100         sldi    r10,r0,3
101         cmpdi   cr6,r6,3
102         andi.   r5,r5,7
103         mtctr   r7
104         subfic  r11,r10,64
105         add     r5,r5,r0
106
107         bt      cr7*4+0,0f
108
109         ld      r9,0(r4)        # 3+2n loads, 2+2n stores
110         ld      r0,8(r4)
111         sld     r6,r9,r10
112         ldu     r9,16(r4)
113         srd     r7,r0,r11
114         sld     r8,r0,r10
115         or      r7,r7,r6
116         blt     cr6,4f
117         ld      r0,8(r4)
118         # s1<< in r8, d0=(s0<<|s1>>) in r7, s3 in r0, s2 in r9, nix in r6 & r12
119         b       2f
120
121 0:      ld      r0,0(r4)        # 4+2n loads, 3+2n stores
122         ldu     r9,8(r4)
123         sld     r8,r0,r10
124         addi    r3,r3,-8
125         blt     cr6,5f
126         ld      r0,8(r4)
127         srd     r12,r9,r11
128         sld     r6,r9,r10
129         ldu     r9,16(r4)
130         or      r12,r8,r12
131         srd     r7,r0,r11
132         sld     r8,r0,r10
133         addi    r3,r3,16
134         beq     cr6,3f
135
136         # d0=(s0<<|s1>>) in r12, s1<< in r6, s2>> in r7, s2<< in r8, s3 in r9
137 1:      or      r7,r7,r6
138         ld      r0,8(r4)
139         std     r12,8(r3)
140 2:      srd     r12,r9,r11
141         sld     r6,r9,r10
142         ldu     r9,16(r4)
143         or      r12,r8,r12
144         stdu    r7,16(r3)
145         srd     r7,r0,r11
146         sld     r8,r0,r10
147         bdnz    1b
148
149 3:      std     r12,8(r3)
150         or      r7,r7,r6
151 4:      std     r7,16(r3)
152 5:      srd     r12,r9,r11
153         or      r12,r8,r12
154         std     r12,24(r3)
155         beq     4f
156         cmpwi   cr1,r5,8
157         addi    r3,r3,32
158         sld     r9,r9,r10
159         ble     cr1,6f
160         ld      r0,8(r4)
161         srd     r7,r0,r11
162         or      r9,r7,r9
163 6:
164         bf      cr7*4+1,1f
165         rotldi  r9,r9,32
166         stw     r9,0(r3)
167         addi    r3,r3,4
168 1:      bf      cr7*4+2,2f
169         rotldi  r9,r9,16
170         sth     r9,0(r3)
171         addi    r3,r3,2
172 2:      bf      cr7*4+3,3f
173         rotldi  r9,r9,8
174         stb     r9,0(r3)
175 3:      ld      r3,-STACKFRAMESIZE+STK_REG(R31)(r1)     /* return dest pointer */
176         blr
177
178 .Ldst_unaligned:
179         PPC_MTOCRF(0x01,r6)             # put #bytes to 8B bdry into cr7
180         subf    r5,r6,r5
181         li      r7,0
182         cmpldi  cr1,r5,16
183         bf      cr7*4+3,1f
184         lbz     r0,0(r4)
185         stb     r0,0(r3)
186         addi    r7,r7,1
187 1:      bf      cr7*4+2,2f
188         lhzx    r0,r7,r4
189         sthx    r0,r7,r3
190         addi    r7,r7,2
191 2:      bf      cr7*4+1,3f
192         lwzx    r0,r7,r4
193         stwx    r0,r7,r3
194 3:      PPC_MTOCRF(0x01,r5)
195         add     r4,r6,r4
196         add     r3,r6,r3
197         b       .Ldst_aligned
198
199 .Lshort_copy:
200         bf      cr7*4+0,1f
201         lwz     r0,0(r4)
202         lwz     r9,4(r4)
203         addi    r4,r4,8
204         stw     r0,0(r3)
205         stw     r9,4(r3)
206         addi    r3,r3,8
207 1:      bf      cr7*4+1,2f
208         lwz     r0,0(r4)
209         addi    r4,r4,4
210         stw     r0,0(r3)
211         addi    r3,r3,4
212 2:      bf      cr7*4+2,3f
213         lhz     r0,0(r4)
214         addi    r4,r4,2
215         sth     r0,0(r3)
216         addi    r3,r3,2
217 3:      bf      cr7*4+3,4f
218         lbz     r0,0(r4)
219         stb     r0,0(r3)
220 4:      ld      r3,-STACKFRAMESIZE+STK_REG(R31)(r1)     /* return dest pointer */
221         blr
222 #endif
223 EXPORT_SYMBOL(memcpy)