arch/arm64/include/asm/alternative.h

   1 #ifndef __ASM_ALTERNATIVE_H
   2 #define __ASM_ALTERNATIVE_H
   3
   4 #include <asm/cpucaps.h>
   5 #include <asm/insn.h>
   6
   7 #define ARM64_CB_PATCH ARM64_NCAPS
   8
   9 #ifndef __ASSEMBLY__
  10
  11 #include <linux/init.h>
  12 #include <linux/types.h>
  13 #include <linux/stddef.h>
  14 #include <linux/stringify.h>
  15
  16 extern int alternatives_applied;
  17
  18 struct alt_instr {
  19         s32 orig_offset;        /* offset to original instruction */
  20         s32 alt_offset;         /* offset to replacement instruction */
  21         u16 cpufeature;         /* cpufeature bit set for replacement */
  22         u8  orig_len;           /* size of original instruction(s) */
  23         u8  alt_len;            /* size of new instruction(s), <= orig_len */
  24 };
  25
  26 typedef void (*alternative_cb_t)(struct alt_instr *alt,
  27                                  __le32 *origptr, __le32 *updptr, int nr_inst);
  28
  29 void __init apply_alternatives_all(void);
  30 void apply_alternatives(void *start, size_t length);
  31
  32 #define ALTINSTR_ENTRY(feature)                                               \
  33         " .word 661b - .\n"                             /* label           */ \
  34         " .word 663f - .\n"                             /* new instruction */ \
  35         " .hword " __stringify(feature) "\n"            /* feature bit     */ \
  36         " .byte 662b-661b\n"                            /* source len      */ \
  37         " .byte 664f-663f\n"                            /* replacement len */
  38
  39 #define ALTINSTR_ENTRY_CB(feature, cb)                                        \
  40         " .word 661b - .\n"                             /* label           */ \
  41         " .word " __stringify(cb) "- .\n"               /* callback */        \
  42         " .hword " __stringify(feature) "\n"            /* feature bit     */ \
  43         " .byte 662b-661b\n"                            /* source len      */ \
  44         " .byte 664f-663f\n"                            /* replacement len */
  45
  46 /*
  47  * alternative assembly primitive:
  48  *
  49  * If any of these .org directive fail, it means that insn1 and insn2
  50  * don't have the same length. This used to be written as
  51  *
  52  * .if ((664b-663b) != (662b-661b))
  53  *      .error "Alternatives instruction length mismatch"
  54  * .endif
  55  *
  56  * but most assemblers die if insn1 or insn2 have a .inst. This should
  57  * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
  58  * containing commit 4e4d08cf7399b606 or c1baaddf8861).
  59  *
  60  * Alternatives with callbacks do not generate replacement instructions.
  61  */
  62 #define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled)     \
  63         ".if "__stringify(cfg_enabled)" == 1\n"                         \
  64         "661:\n\t"                                                      \
  65         oldinstr "\n"                                                   \
  66         "662:\n"                                                        \
  67         ".pushsection .altinstructions,\"a\"\n"                         \
  68         ALTINSTR_ENTRY(feature)                                         \
  69         ".popsection\n"                                                 \
  70         ".pushsection .altinstr_replacement, \"a\"\n"                   \
  71         "663:\n\t"                                                      \
  72         newinstr "\n"                                                   \
  73         "664:\n\t"                                                      \
  74         ".popsection\n\t"                                               \
  75         ".org   . - (664b-663b) + (662b-661b)\n\t"                      \
  76         ".org   . - (662b-661b) + (664b-663b)\n"                        \
  77         ".endif\n"
  78
  79 #define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb)        \
  80         ".if "__stringify(cfg_enabled)" == 1\n"                         \
  81         "661:\n\t"                                                      \
  82         oldinstr "\n"                                                   \
  83         "662:\n"                                                        \
  84         ".pushsection .altinstructions,\"a\"\n"                         \
  85         ALTINSTR_ENTRY_CB(feature, cb)                                  \
  86         ".popsection\n"                                                 \
  87         "663:\n\t"                                                      \
  88         "664:\n\t"                                                      \
  89         ".endif\n"
  90
  91 #define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
  92         __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
  93
  94 #define ALTERNATIVE_CB(oldinstr, cb) \
  95         __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)
  96 #else
  97
  98 #include <asm/assembler.h>
  99
 100 .macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
 101         .word \orig_offset - .
 102         .word \alt_offset - .
 103         .hword \feature
 104         .byte \orig_len
 105         .byte \alt_len
 106 .endm
 107
 108 .macro alternative_insn insn1, insn2, cap, enable = 1
 109         .if \enable
 110 661:    \insn1
 111 662:    .pushsection .altinstructions, "a"
 112         altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
 113         .popsection
 114         .pushsection .altinstr_replacement, "ax"
 115 663:    \insn2
 116 664:    .popsection
 117         .org    . - (664b-663b) + (662b-661b)
 118         .org    . - (662b-661b) + (664b-663b)
 119         .endif
 120 .endm
 121
 122 /*
 123  * Alternative sequences
 124  *
 125  * The code for the case where the capability is not present will be
 126  * assembled and linked as normal. There are no restrictions on this
 127  * code.
 128  *
 129  * The code for the case where the capability is present will be
 130  * assembled into a special section to be used for dynamic patching.
 131  * Code for that case must:
 132  *
 133  * 1. Be exactly the same length (in bytes) as the default code
 134  *    sequence.
 135  *
 136  * 2. Not contain a branch target that is used outside of the
 137  *    alternative sequence it is defined in (branches into an
 138  *    alternative sequence are not fixed up).
 139  */
 140
 141 /*
 142  * Begin an alternative code sequence.
 143  */
 144 .macro alternative_if_not cap
 145         .set .Lasm_alt_mode, 0
 146         .pushsection .altinstructions, "a"
 147         altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
 148         .popsection
 149 661:
 150 .endm
 151
 152 .macro alternative_if cap
 153         .set .Lasm_alt_mode, 1
 154         .pushsection .altinstructions, "a"
 155         altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
 156         .popsection
 157         .pushsection .altinstr_replacement, "ax"
 158         .align 2        /* So GAS knows label 661 is suitably aligned */
 159 661:
 160 .endm
 161
 162 .macro alternative_cb cb
 163         .set .Lasm_alt_mode, 0
 164         .pushsection .altinstructions, "a"
 165         altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
 166         .popsection
 167 661:
 168 .endm
 169
 170 /*
 171  * Provide the other half of the alternative code sequence.
 172  */
 173 .macro alternative_else
 174 662:
 175         .if .Lasm_alt_mode==0
 176         .pushsection .altinstr_replacement, "ax"
 177         .else
 178         .popsection
 179         .endif
 180 663:
 181 .endm
 182
 183 /*
 184  * Complete an alternative code sequence.
 185  */
 186 .macro alternative_endif
 187 664:
 188         .if .Lasm_alt_mode==0
 189         .popsection
 190         .endif
 191         .org    . - (664b-663b) + (662b-661b)
 192         .org    . - (662b-661b) + (664b-663b)
 193 .endm
 194
 195 /*
 196  * Callback-based alternative epilogue
 197  */
 198 .macro alternative_cb_end
 199 662:
 200 .endm
 201
 202 /*
 203  * Provides a trivial alternative or default sequence consisting solely
 204  * of NOPs. The number of NOPs is chosen automatically to match the
 205  * previous case.
 206  */
 207 .macro alternative_else_nop_endif
 208 alternative_else
 209         nops    (662b-661b) / AARCH64_INSN_SIZE
 210 alternative_endif
 211 .endm
 212
 213 #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)   \
 214         alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
 215
 216 .macro user_alt, label, oldinstr, newinstr, cond
 217 9999:   alternative_insn "\oldinstr", "\newinstr", \cond
 218         _asm_extable 9999b, \label
 219 .endm
 220
 221 /*
 222  * Generate the assembly for UAO alternatives with exception table entries.
 223  * This is complicated as there is no post-increment or pair versions of the
 224  * unprivileged instructions, and USER() only works for single instructions.
 225  */
 226 #ifdef CONFIG_ARM64_UAO
 227         .macro uao_ldp l, reg1, reg2, addr, post_inc
 228                 alternative_if_not ARM64_HAS_UAO
 229 8888:                   ldp     \reg1, \reg2, [\addr], \post_inc;
 230 8889:                   nop;
 231                         nop;
 232                 alternative_else
 233                         ldtr    \reg1, [\addr];
 234                         ldtr    \reg2, [\addr, #8];
 235                         add     \addr, \addr, \post_inc;
 236                 alternative_endif
 237
 238                 _asm_extable    8888b,\l;
 239                 _asm_extable    8889b,\l;
 240         .endm
 241
 242         .macro uao_stp l, reg1, reg2, addr, post_inc
 243                 alternative_if_not ARM64_HAS_UAO
 244 8888:                   stp     \reg1, \reg2, [\addr], \post_inc;
 245 8889:                   nop;
 246                         nop;
 247                 alternative_else
 248                         sttr    \reg1, [\addr];
 249                         sttr    \reg2, [\addr, #8];
 250                         add     \addr, \addr, \post_inc;
 251                 alternative_endif
 252
 253                 _asm_extable    8888b,\l;
 254                 _asm_extable    8889b,\l;
 255         .endm
 256
 257         .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
 258                 alternative_if_not ARM64_HAS_UAO
 259 8888:                   \inst   \reg, [\addr], \post_inc;
 260                         nop;
 261                 alternative_else
 262                         \alt_inst       \reg, [\addr];
 263                         add             \addr, \addr, \post_inc;
 264                 alternative_endif
 265
 266                 _asm_extable    8888b,\l;
 267         .endm
 268 #else
 269         .macro uao_ldp l, reg1, reg2, addr, post_inc
 270                 USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
 271         .endm
 272         .macro uao_stp l, reg1, reg2, addr, post_inc
 273                 USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
 274         .endm
 275         .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
 276                 USER(\l, \inst \reg, [\addr], \post_inc)
 277         .endm
 278 #endif
 279
 280 #endif  /*  __ASSEMBLY__  */
 281
 282 /*
 283  * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
 284  *
 285  * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
 286  * N.B. If CONFIG_FOO is specified, but not selected, the whole block
 287  *      will be omitted, including oldinstr.
 288  */
 289 #define ALTERNATIVE(oldinstr, newinstr, ...)   \
 290         _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
 291
 292 #endif /* __ASM_ALTERNATIVE_H */