2 * Copyright (C) 2013 ARM Ltd.
3 * Copyright (C) 2013 Linaro.
5 * This code is based on glibc cortex strings work originally authored by Linaro
6 * and re-licensed under GPLv2 for the Linux kernel. The original code can
9 * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
10 * files/head:/src/aarch64/
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program. If not, see <http://www.gnu.org/licenses/>.
25 #include <linux/linkage.h>
26 #include <asm/assembler.h>
27 #include <asm/cache.h>
30 * Fill in the buffer with character c (alignment handled by the hardware)
59 mov dst, dstin /* Preserve return value. */
61 orr A_lw, A_lw, A_lw, lsl #8
62 orr A_lw, A_lw, A_lw, lsl #16
63 orr A_l, A_l, A_l, lsl #32
67 /*All store maybe are non-aligned..*/
83 /*Whether the start address is aligned with 16.*/
88 * The count is not less than 16, we can use stp to store the start 16 bytes,
89 * then adjust the dst aligned with 16.This process will make the current
90 * memory address at alignment boundary.
92 stp A_l, A_l, [dst] /*non-aligned store..*/
93 /*make the dst aligned..*/
94 sub count, count, tmp2
104 ands tmp1, count, #0x30
109 stp A_l, A_l, [dst], #16
111 stp A_l, A_l, [dst], #16
113 stp A_l, A_l, [dst], #16
115 * The last store length is less than 16,use stp to write last 16 bytes.
116 * It will lead some bytes written twice and the access is non-aligned.
119 ands count, count, #15
122 stp A_l, A_l, [dst, #-16] /* Repeat some/all of last store. */
127 * Critical loop. Start at a new cache line boundary. Assuming
128 * 64 bytes per line, this ensures the entire loop is in one line.
130 .p2align L1_CACHE_SHIFT
132 sub dst, dst, #16/* Pre-bias. */
133 sub count, count, #64
135 stp A_l, A_l, [dst, #16]
136 stp A_l, A_l, [dst, #32]
137 stp A_l, A_l, [dst, #48]
138 stp A_l, A_l, [dst, #64]!
139 subs count, count, #64
148 * For zeroing memory, check to see if we can use the ZVA feature to
149 * zero entire 'cache' lines.
155 * For zeroing small amounts of memory, it's not worth setting up
156 * the line-clear code.
159 b.lt .Lnot_short /*count is at least 128 bytes*/
162 tbnz tmp1, #4, .Lnot_short
164 and zva_len, tmp1w, #15 /* Safety: other bits reserved. */
165 lsl zva_len, tmp3w, zva_len
167 ands tmp3w, zva_len, #63
169 * ensure the zva_len is not less than 64.
170 * It is not meaningful to use ZVA if the block size is less than 64.
175 * Compute how far we need to go to become suitably aligned. We're
176 * already at quad-word alignment.
179 b.lt .Lnot_short /* Not enough to reach alignment. */
180 sub zva_bits_x, zva_len_x, #1
182 ands tmp2, tmp2, zva_bits_x
183 b.eq 2f /* Already aligned. */
184 /* Not aligned, check that there's enough to copy after alignment.*/
185 sub tmp1, count, tmp2
187 * grantee the remain length to be ZVA is bigger than 64,
188 * avoid to make the 2f's process over mem range.*/
190 ccmp tmp1, zva_len_x, #8, ge /* NZCV=0b1000 */
193 * We know that there's at least 64 bytes to zero and that it's safe
194 * to overrun by 64 bytes.
199 stp A_l, A_l, [dst, #16]
200 stp A_l, A_l, [dst, #32]
202 stp A_l, A_l, [dst, #48]
205 /* We've overrun a bit, so adjust dst downwards.*/
208 sub count, count, zva_len_x
211 add dst, dst, zva_len_x
212 subs count, count, zva_len_x
214 ands count, count, zva_bits_x
215 b.ne .Ltail_maybe_long