arch/arm64/include/asm/alternative.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef __ASM_ALTERNATIVE_H
   3 #define __ASM_ALTERNATIVE_H
   4
   5 #include <asm/cpucaps.h>
   6 #include <asm/insn.h>
   7
   8 #define ARM64_CB_PATCH ARM64_NCAPS
   9
  10 #ifndef __ASSEMBLY__
  11
  12 #include <linux/init.h>
  13 #include <linux/types.h>
  14 #include <linux/stddef.h>
  15 #include <linux/stringify.h>
  16
  17 extern int alternatives_applied;
  18
  19 struct alt_instr {
  20         s32 orig_offset;        /* offset to original instruction */
  21         s32 alt_offset;         /* offset to replacement instruction */
  22         u16 cpufeature;         /* cpufeature bit set for replacement */
  23         u8  orig_len;           /* size of original instruction(s) */
  24         u8  alt_len;            /* size of new instruction(s), <= orig_len */
  25 };
  26
  27 typedef void (*alternative_cb_t)(struct alt_instr *alt,
  28                                  __le32 *origptr, __le32 *updptr, int nr_inst);
  29
  30 void __init apply_alternatives_all(void);
  31 void apply_alternatives(void *start, size_t length);
  32
  33 #define ALTINSTR_ENTRY(feature)                                               \
  34         " .word 661b - .\n"                             /* label           */ \
  35         " .word 663f - .\n"                             /* new instruction */ \
  36         " .hword " __stringify(feature) "\n"            /* feature bit     */ \
  37         " .byte 662b-661b\n"                            /* source len      */ \
  38         " .byte 664f-663f\n"                            /* replacement len */
  39
  40 #define ALTINSTR_ENTRY_CB(feature, cb)                                        \
  41         " .word 661b - .\n"                             /* label           */ \
  42         " .word " __stringify(cb) "- .\n"               /* callback */        \
  43         " .hword " __stringify(feature) "\n"            /* feature bit     */ \
  44         " .byte 662b-661b\n"                            /* source len      */ \
  45         " .byte 664f-663f\n"                            /* replacement len */
  46
  47 /*
  48  * alternative assembly primitive:
  49  *
  50  * If any of these .org directive fail, it means that insn1 and insn2
  51  * don't have the same length. This used to be written as
  52  *
  53  * .if ((664b-663b) != (662b-661b))
  54  *      .error "Alternatives instruction length mismatch"
  55  * .endif
  56  *
  57  * but most assemblers die if insn1 or insn2 have a .inst. This should
  58  * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
  59  * containing commit 4e4d08cf7399b606 or c1baaddf8861).
  60  *
  61  * Alternatives with callbacks do not generate replacement instructions.
  62  */
  63 #define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled)     \
  64         ".if "__stringify(cfg_enabled)" == 1\n"                         \
  65         "661:\n\t"                                                      \
  66         oldinstr "\n"                                                   \
  67         "662:\n"                                                        \
  68         ".pushsection .altinstructions,\"a\"\n"                         \
  69         ALTINSTR_ENTRY(feature)                                         \
  70         ".popsection\n"                                                 \
  71         ".subsection 1\n"                                               \
  72         "663:\n\t"                                                      \
  73         newinstr "\n"                                                   \
  74         "664:\n\t"                                                      \
  75         ".org   . - (664b-663b) + (662b-661b)\n\t"                      \
  76         ".org   . - (662b-661b) + (664b-663b)\n\t"                      \
  77         ".previous\n"                                                   \
  78         ".endif\n"
  79
  80 #define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb)        \
  81         ".if "__stringify(cfg_enabled)" == 1\n"                         \
  82         "661:\n\t"                                                      \
  83         oldinstr "\n"                                                   \
  84         "662:\n"                                                        \
  85         ".pushsection .altinstructions,\"a\"\n"                         \
  86         ALTINSTR_ENTRY_CB(feature, cb)                                  \
  87         ".popsection\n"                                                 \
  88         "663:\n\t"                                                      \
  89         "664:\n\t"                                                      \
  90         ".endif\n"
  91
  92 #define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
  93         __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
  94
  95 #define ALTERNATIVE_CB(oldinstr, cb) \
  96         __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)
  97 #else
  98
  99 #include <asm/assembler.h>
 100
 101 .macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
 102         .word \orig_offset - .
 103         .word \alt_offset - .
 104         .hword \feature
 105         .byte \orig_len
 106         .byte \alt_len
 107 .endm
 108
 109 .macro alternative_insn insn1, insn2, cap, enable = 1
 110         .if \enable
 111 661:    \insn1
 112 662:    .pushsection .altinstructions, "a"
 113         altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
 114         .popsection
 115         .subsection 1
 116 663:    \insn2
 117 664:    .org    . - (664b-663b) + (662b-661b)
 118         .org    . - (662b-661b) + (664b-663b)
 119         .previous
 120         .endif
 121 .endm
 122
 123 /*
 124  * Alternative sequences
 125  *
 126  * The code for the case where the capability is not present will be
 127  * assembled and linked as normal. There are no restrictions on this
 128  * code.
 129  *
 130  * The code for the case where the capability is present will be
 131  * assembled into a special section to be used for dynamic patching.
 132  * Code for that case must:
 133  *
 134  * 1. Be exactly the same length (in bytes) as the default code
 135  *    sequence.
 136  *
 137  * 2. Not contain a branch target that is used outside of the
 138  *    alternative sequence it is defined in (branches into an
 139  *    alternative sequence are not fixed up).
 140  */
 141
 142 /*
 143  * Begin an alternative code sequence.
 144  */
 145 .macro alternative_if_not cap
 146         .set .Lasm_alt_mode, 0
 147         .pushsection .altinstructions, "a"
 148         altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
 149         .popsection
 150 661:
 151 .endm
 152
 153 .macro alternative_if cap
 154         .set .Lasm_alt_mode, 1
 155         .pushsection .altinstructions, "a"
 156         altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
 157         .popsection
 158         .subsection 1
 159         .align 2        /* So GAS knows label 661 is suitably aligned */
 160 661:
 161 .endm
 162
 163 .macro alternative_cb cb
 164         .set .Lasm_alt_mode, 0
 165         .pushsection .altinstructions, "a"
 166         altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
 167         .popsection
 168 661:
 169 .endm
 170
 171 /*
 172  * Provide the other half of the alternative code sequence.
 173  */
 174 .macro alternative_else
 175 662:
 176         .if .Lasm_alt_mode==0
 177         .subsection 1
 178         .else
 179         .previous
 180         .endif
 181 663:
 182 .endm
 183
 184 /*
 185  * Complete an alternative code sequence.
 186  */
 187 .macro alternative_endif
 188 664:
 189         .org    . - (664b-663b) + (662b-661b)
 190         .org    . - (662b-661b) + (664b-663b)
 191         .if .Lasm_alt_mode==0
 192         .previous
 193         .endif
 194 .endm
 195
 196 /*
 197  * Callback-based alternative epilogue
 198  */
 199 .macro alternative_cb_end
 200 662:
 201 .endm
 202
 203 /*
 204  * Provides a trivial alternative or default sequence consisting solely
 205  * of NOPs. The number of NOPs is chosen automatically to match the
 206  * previous case.
 207  */
 208 .macro alternative_else_nop_endif
 209 alternative_else
 210         nops    (662b-661b) / AARCH64_INSN_SIZE
 211 alternative_endif
 212 .endm
 213
 214 #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)   \
 215         alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
 216
 217 .macro user_alt, label, oldinstr, newinstr, cond
 218 9999:   alternative_insn "\oldinstr", "\newinstr", \cond
 219         _asm_extable 9999b, \label
 220 .endm
 221
 222 /*
 223  * Generate the assembly for UAO alternatives with exception table entries.
 224  * This is complicated as there is no post-increment or pair versions of the
 225  * unprivileged instructions, and USER() only works for single instructions.
 226  */
 227 #ifdef CONFIG_ARM64_UAO
 228         .macro uao_ldp l, reg1, reg2, addr, post_inc
 229                 alternative_if_not ARM64_HAS_UAO
 230 8888:                   ldp     \reg1, \reg2, [\addr], \post_inc;
 231 8889:                   nop;
 232                         nop;
 233                 alternative_else
 234                         ldtr    \reg1, [\addr];
 235                         ldtr    \reg2, [\addr, #8];
 236                         add     \addr, \addr, \post_inc;
 237                 alternative_endif
 238
 239                 _asm_extable    8888b,\l;
 240                 _asm_extable    8889b,\l;
 241         .endm
 242
 243         .macro uao_stp l, reg1, reg2, addr, post_inc
 244                 alternative_if_not ARM64_HAS_UAO
 245 8888:                   stp     \reg1, \reg2, [\addr], \post_inc;
 246 8889:                   nop;
 247                         nop;
 248                 alternative_else
 249                         sttr    \reg1, [\addr];
 250                         sttr    \reg2, [\addr, #8];
 251                         add     \addr, \addr, \post_inc;
 252                 alternative_endif
 253
 254                 _asm_extable    8888b,\l;
 255                 _asm_extable    8889b,\l;
 256         .endm
 257
 258         .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
 259                 alternative_if_not ARM64_HAS_UAO
 260 8888:                   \inst   \reg, [\addr], \post_inc;
 261                         nop;
 262                 alternative_else
 263                         \alt_inst       \reg, [\addr];
 264                         add             \addr, \addr, \post_inc;
 265                 alternative_endif
 266
 267                 _asm_extable    8888b,\l;
 268         .endm
 269 #else
 270         .macro uao_ldp l, reg1, reg2, addr, post_inc
 271                 USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
 272         .endm
 273         .macro uao_stp l, reg1, reg2, addr, post_inc
 274                 USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
 275         .endm
 276         .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
 277                 USER(\l, \inst \reg, [\addr], \post_inc)
 278         .endm
 279 #endif
 280
 281 #endif  /*  __ASSEMBLY__  */
 282
 283 /*
 284  * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
 285  *
 286  * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
 287  * N.B. If CONFIG_FOO is specified, but not selected, the whole block
 288  *      will be omitted, including oldinstr.
 289  */
 290 #define ALTERNATIVE(oldinstr, newinstr, ...)   \
 291         _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
 292
 293 #endif /* __ASM_ALTERNATIVE_H */