arch/metag/lib/usercopy.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * User address space access functions.
   4  * The non-inlined parts of asm-metag/uaccess.h are here.
   5  *
   6  * Copyright (C) 2006, Imagination Technologies.
   7  * Copyright (C) 2000, Axis Communications AB.
   8  *
   9  * Written by Hans-Peter Nilsson.
  10  * Pieces used from memcpy, originally by Kenny Ranerup long time ago.
  11  * Modified for Meta by Will Newton.
  12  */
  13
  14 #include <linux/export.h>
  15 #include <linux/uaccess.h>
  16 #include <asm/cache.h>                  /* def of L1_CACHE_BYTES */
  17
  18 #define USE_RAPF
  19 #define RAPF_MIN_BUF_SIZE       (3*L1_CACHE_BYTES)
  20
  21
  22 /* The "double write" in this code is because the Meta will not fault
  23  * immediately unless the memory pipe is forced to by e.g. a data stall or
  24  * another memory op. The second write should be discarded by the write
  25  * combiner so should have virtually no cost.
  26  */
  27
  28 #define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \
  29         asm volatile (                                           \
  30                 COPY                                             \
  31                 "1:\n"                                           \
  32                 "       .section .fixup,\"ax\"\n"                \
  33                 FIXUP                                            \
  34                 "       MOVT    D1Ar1,#HI(1b)\n"                 \
  35                 "       JUMP    D1Ar1,#LO(1b)\n"                 \
  36                 "       .previous\n"                             \
  37                 "       .section __ex_table,\"a\"\n"             \
  38                 TENTRY                                           \
  39                 "       .previous\n"                             \
  40                 : "=r" (to), "=r" (from), "=r" (ret)             \
  41                 : "0" (to), "1" (from), "2" (ret)                \
  42                 : "D1Ar1", "memory")
  43
  44
  45 #define __asm_copy_to_user_1(to, from, ret)     \
  46         __asm_copy_user_cont(to, from, ret,     \
  47                 "       GETB D1Ar1,[%1++]\n"    \
  48                 "       SETB [%0],D1Ar1\n"      \
  49                 "2:     SETB [%0++],D1Ar1\n",   \
  50                 "3:     ADD  %2,%2,#1\n",       \
  51                 "       .long 2b,3b\n")
  52
  53 #define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
  54         __asm_copy_user_cont(to, from, ret,             \
  55                 "       GETW D1Ar1,[%1++]\n"            \
  56                 "       SETW [%0],D1Ar1\n"              \
  57                 "2:     SETW [%0++],D1Ar1\n" COPY,      \
  58                 "3:     ADD  %2,%2,#2\n" FIXUP,         \
  59                 "       .long 2b,3b\n" TENTRY)
  60
  61 #define __asm_copy_to_user_2(to, from, ret) \
  62         __asm_copy_to_user_2x_cont(to, from, ret, "", "", "")
  63
  64 #define __asm_copy_to_user_3(to, from, ret) \
  65         __asm_copy_to_user_2x_cont(to, from, ret,       \
  66                 "       GETB D1Ar1,[%1++]\n"            \
  67                 "       SETB [%0],D1Ar1\n"              \
  68                 "4:     SETB [%0++],D1Ar1\n",           \
  69                 "5:     ADD  %2,%2,#1\n",               \
  70                 "       .long 4b,5b\n")
  71
  72 #define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
  73         __asm_copy_user_cont(to, from, ret,             \
  74                 "       GETD D1Ar1,[%1++]\n"            \
  75                 "       SETD [%0],D1Ar1\n"              \
  76                 "2:     SETD [%0++],D1Ar1\n" COPY,      \
  77                 "3:     ADD  %2,%2,#4\n" FIXUP,         \
  78                 "       .long 2b,3b\n" TENTRY)
  79
  80 #define __asm_copy_to_user_4(to, from, ret) \
  81         __asm_copy_to_user_4x_cont(to, from, ret, "", "", "")
  82
  83 #define __asm_copy_to_user_5(to, from, ret) \
  84         __asm_copy_to_user_4x_cont(to, from, ret,       \
  85                 "       GETB D1Ar1,[%1++]\n"            \
  86                 "       SETB [%0],D1Ar1\n"              \
  87                 "4:     SETB [%0++],D1Ar1\n",           \
  88                 "5:     ADD  %2,%2,#1\n",               \
  89                 "       .long 4b,5b\n")
  90
  91 #define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
  92         __asm_copy_to_user_4x_cont(to, from, ret,       \
  93                 "       GETW D1Ar1,[%1++]\n"            \
  94                 "       SETW [%0],D1Ar1\n"              \
  95                 "4:     SETW [%0++],D1Ar1\n" COPY,      \
  96                 "5:     ADD  %2,%2,#2\n" FIXUP,         \
  97                 "       .long 4b,5b\n" TENTRY)
  98
  99 #define __asm_copy_to_user_6(to, from, ret) \
 100         __asm_copy_to_user_6x_cont(to, from, ret, "", "", "")
 101
 102 #define __asm_copy_to_user_7(to, from, ret) \
 103         __asm_copy_to_user_6x_cont(to, from, ret,       \
 104                 "       GETB D1Ar1,[%1++]\n"            \
 105                 "       SETB [%0],D1Ar1\n"              \
 106                 "6:     SETB [%0++],D1Ar1\n",           \
 107                 "7:     ADD  %2,%2,#1\n",               \
 108                 "       .long 6b,7b\n")
 109
 110 #define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 111         __asm_copy_to_user_4x_cont(to, from, ret,       \
 112                 "       GETD D1Ar1,[%1++]\n"            \
 113                 "       SETD [%0],D1Ar1\n"              \
 114                 "4:     SETD [%0++],D1Ar1\n" COPY,      \
 115                 "5:     ADD  %2,%2,#4\n"  FIXUP,        \
 116                 "       .long 4b,5b\n" TENTRY)
 117
 118 #define __asm_copy_to_user_8(to, from, ret) \
 119         __asm_copy_to_user_8x_cont(to, from, ret, "", "", "")
 120
 121 #define __asm_copy_to_user_9(to, from, ret) \
 122         __asm_copy_to_user_8x_cont(to, from, ret,       \
 123                 "       GETB D1Ar1,[%1++]\n"            \
 124                 "       SETB [%0],D1Ar1\n"              \
 125                 "6:     SETB [%0++],D1Ar1\n",           \
 126                 "7:     ADD  %2,%2,#1\n",               \
 127                 "       .long 6b,7b\n")
 128
 129 #define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 130         __asm_copy_to_user_8x_cont(to, from, ret,       \
 131                 "       GETW D1Ar1,[%1++]\n"            \
 132                 "       SETW [%0],D1Ar1\n"              \
 133                 "6:     SETW [%0++],D1Ar1\n" COPY,      \
 134                 "7:     ADD  %2,%2,#2\n" FIXUP,         \
 135                 "       .long 6b,7b\n" TENTRY)
 136
 137 #define __asm_copy_to_user_10(to, from, ret) \
 138         __asm_copy_to_user_10x_cont(to, from, ret, "", "", "")
 139
 140 #define __asm_copy_to_user_11(to, from, ret) \
 141         __asm_copy_to_user_10x_cont(to, from, ret,      \
 142                 "       GETB D1Ar1,[%1++]\n"            \
 143                 "       SETB [%0],D1Ar1\n"              \
 144                 "8:     SETB [%0++],D1Ar1\n",           \
 145                 "9:     ADD  %2,%2,#1\n",               \
 146                 "       .long 8b,9b\n")
 147
 148 #define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 149         __asm_copy_to_user_8x_cont(to, from, ret,       \
 150                 "       GETD D1Ar1,[%1++]\n"            \
 151                 "       SETD [%0],D1Ar1\n"              \
 152                 "6:     SETD [%0++],D1Ar1\n" COPY,      \
 153                 "7:     ADD  %2,%2,#4\n" FIXUP,         \
 154                 "       .long 6b,7b\n" TENTRY)
 155 #define __asm_copy_to_user_12(to, from, ret) \
 156         __asm_copy_to_user_12x_cont(to, from, ret, "", "", "")
 157
 158 #define __asm_copy_to_user_13(to, from, ret) \
 159         __asm_copy_to_user_12x_cont(to, from, ret,      \
 160                 "       GETB D1Ar1,[%1++]\n"            \
 161                 "       SETB [%0],D1Ar1\n"              \
 162                 "8:     SETB [%0++],D1Ar1\n",           \
 163                 "9:     ADD  %2,%2,#1\n",               \
 164                 "       .long 8b,9b\n")
 165
 166 #define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 167         __asm_copy_to_user_12x_cont(to, from, ret,      \
 168                 "       GETW D1Ar1,[%1++]\n"            \
 169                 "       SETW [%0],D1Ar1\n"              \
 170                 "8:     SETW [%0++],D1Ar1\n" COPY,      \
 171                 "9:     ADD  %2,%2,#2\n" FIXUP,         \
 172                 "       .long 8b,9b\n" TENTRY)
 173
 174 #define __asm_copy_to_user_14(to, from, ret) \
 175         __asm_copy_to_user_14x_cont(to, from, ret, "", "", "")
 176
 177 #define __asm_copy_to_user_15(to, from, ret) \
 178         __asm_copy_to_user_14x_cont(to, from, ret,      \
 179                 "       GETB D1Ar1,[%1++]\n"            \
 180                 "       SETB [%0],D1Ar1\n"              \
 181                 "10:    SETB [%0++],D1Ar1\n",           \
 182                 "11:    ADD  %2,%2,#1\n",               \
 183                 "       .long 10b,11b\n")
 184
 185 #define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 186         __asm_copy_to_user_12x_cont(to, from, ret,      \
 187                 "       GETD D1Ar1,[%1++]\n"            \
 188                 "       SETD [%0],D1Ar1\n"              \
 189                 "8:     SETD [%0++],D1Ar1\n" COPY,      \
 190                 "9:     ADD  %2,%2,#4\n" FIXUP,         \
 191                 "       .long 8b,9b\n" TENTRY)
 192
 193 #define __asm_copy_to_user_16(to, from, ret) \
 194                 __asm_copy_to_user_16x_cont(to, from, ret, "", "", "")
 195
 196 #define __asm_copy_to_user_8x64(to, from, ret) \
 197         asm volatile (                                  \
 198                 "       GETL D0Ar2,D1Ar1,[%1++]\n"      \
 199                 "       SETL [%0],D0Ar2,D1Ar1\n"        \
 200                 "2:     SETL [%0++],D0Ar2,D1Ar1\n"      \
 201                 "1:\n"                                  \
 202                 "       .section .fixup,\"ax\"\n"       \
 203                 "3:     ADD  %2,%2,#8\n"                \
 204                 "       MOVT    D0Ar2,#HI(1b)\n"        \
 205                 "       JUMP    D0Ar2,#LO(1b)\n"        \
 206                 "       .previous\n"                    \
 207                 "       .section __ex_table,\"a\"\n"    \
 208                 "       .long 2b,3b\n"                  \
 209                 "       .previous\n"                    \
 210                 : "=r" (to), "=r" (from), "=r" (ret)    \
 211                 : "0" (to), "1" (from), "2" (ret)       \
 212                 : "D1Ar1", "D0Ar2", "memory")
 213
 214 /*
 215  *      optimized copying loop using RAPF when 64 bit aligned
 216  *
 217  *      n               will be automatically decremented inside the loop
 218  *      ret             will be left intact. if error occurs we will rewind
 219  *                      so that the original non optimized code will fill up
 220  *                      this value correctly.
 221  *
 222  *      on fault:
 223  *              >       n will hold total number of uncopied bytes
 224  *
 225  *              >       {'to','from'} will be rewind back so that
 226  *                      the non-optimized code will do the proper fix up
 227  *
 228  *      DCACHE drops the cacheline which helps in reducing cache
 229  *      pollution.
 230  *
 231  *      We introduce an extra SETL at the end of the loop to
 232  *      ensure we don't fall off the loop before we catch all
 233  *      erros.
 234  *
 235  *      NOTICE:
 236  *              LSM_STEP in TXSTATUS must be cleared in fix up code.
 237  *              since we're using M{S,G}ETL, a fault might happen at
 238  *              any address in the middle of M{S,G}ETL causing
 239  *              the value of LSM_STEP to be incorrect which can
 240  *              cause subsequent use of M{S,G}ET{L,D} to go wrong.
 241  *              ie: if LSM_STEP was 1 when a fault occurs, the
 242  *              next call to M{S,G}ET{L,D} will skip the first
 243  *              copy/getting as it think that the first 1 has already
 244  *              been done.
 245  *
 246  */
 247 #define __asm_copy_user_64bit_rapf_loop(                                \
 248                 to, from, ret, n, id, FIXUP)                            \
 249         asm volatile (                                                  \
 250                         ".balign 8\n"                                   \
 251                 "       MOV     RAPF, %1\n"                             \
 252                 "       MSETL   [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \
 253                 "       MOV     D0Ar6, #0\n"                            \
 254                 "       LSR     D1Ar5, %3, #6\n"                        \
 255                 "       SUB     TXRPT, D1Ar5, #2\n"                     \
 256                 "       MOV     RAPF, %1\n"                             \
 257                 "$Lloop"id":\n"                                         \
 258                 "       ADD     RAPF, %1, #64\n"                        \
 259                 "21:    MGETL   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 260                 "22:    MSETL   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 261                 "23:    SUB     %3, %3, #32\n"                          \
 262                 "24:    MGETL   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 263                 "25:    MSETL   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 264                 "26:    SUB     %3, %3, #32\n"                          \
 265                 "       DCACHE  [%1+#-64], D0Ar6\n"                     \
 266                 "       BR      $Lloop"id"\n"                           \
 267                                                                         \
 268                 "       MOV     RAPF, %1\n"                             \
 269                 "27:    MGETL   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 270                 "28:    MSETL   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 271                 "29:    SUB     %3, %3, #32\n"                          \
 272                 "30:    MGETL   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 273                 "31:    MSETL   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 274                 "32:    SETL    [%0+#-8], D0.7, D1.7\n"                 \
 275                 "       SUB     %3, %3, #32\n"                          \
 276                 "1:     DCACHE  [%1+#-64], D0Ar6\n"                     \
 277                 "       GETL    D0Ar6, D1Ar5, [A0StP+#-40]\n"           \
 278                 "       GETL    D0FrT, D1RtP, [A0StP+#-32]\n"           \
 279                 "       GETL    D0.5, D1.5, [A0StP+#-24]\n"             \
 280                 "       GETL    D0.6, D1.6, [A0StP+#-16]\n"             \
 281                 "       GETL    D0.7, D1.7, [A0StP+#-8]\n"              \
 282                 "       SUB     A0StP, A0StP, #40\n"                    \
 283                 "       .section .fixup,\"ax\"\n"                       \
 284                 "3:     MOV     D0Ar2, TXSTATUS\n"                      \
 285                 "       MOV     D1Ar1, TXSTATUS\n"                      \
 286                 "       AND     D1Ar1, D1Ar1, #0xFFFFF8FF\n"            \
 287                 "       MOV     TXSTATUS, D1Ar1\n"                      \
 288                         FIXUP                                           \
 289                 "       MOVT    D0Ar2, #HI(1b)\n"                       \
 290                 "       JUMP    D0Ar2, #LO(1b)\n"                       \
 291                 "       .previous\n"                                    \
 292                 "       .section __ex_table,\"a\"\n"                    \
 293                 "       .long 21b,3b\n"                                 \
 294                 "       .long 22b,3b\n"                                 \
 295                 "       .long 23b,3b\n"                                 \
 296                 "       .long 24b,3b\n"                                 \
 297                 "       .long 25b,3b\n"                                 \
 298                 "       .long 26b,3b\n"                                 \
 299                 "       .long 27b,3b\n"                                 \
 300                 "       .long 28b,3b\n"                                 \
 301                 "       .long 29b,3b\n"                                 \
 302                 "       .long 30b,3b\n"                                 \
 303                 "       .long 31b,3b\n"                                 \
 304                 "       .long 32b,3b\n"                                 \
 305                 "       .previous\n"                                    \
 306                 : "=r" (to), "=r" (from), "=r" (ret), "=d" (n)          \
 307                 : "0" (to), "1" (from), "2" (ret), "3" (n)              \
 308                 : "D1Ar1", "D0Ar2", "cc", "memory")
 309
 310 /*      rewind 'to' and 'from'  pointers when a fault occurs
 311  *
 312  *      Rationale:
 313  *              A fault always occurs on writing to user buffer. A fault
 314  *              is at a single address, so we need to rewind by only 4
 315  *              bytes.
 316  *              Since we do a complete read from kernel buffer before
 317  *              writing, we need to rewind it also. The amount to be
 318  *              rewind equals the number of faulty writes in MSETD
 319  *              which is: [4 - (LSM_STEP-1)]*8
 320  *              LSM_STEP is bits 10:8 in TXSTATUS which is already read
 321  *              and stored in D0Ar2
 322  *
 323  *              NOTE: If a fault occurs at the last operation in M{G,S}ETL
 324  *                      LSM_STEP will be 0. ie: we do 4 writes in our case, if
 325  *                      a fault happens at the 4th write, LSM_STEP will be 0
 326  *                      instead of 4. The code copes with that.
 327  *
 328  *              n is updated by the number of successful writes, which is:
 329  *              n = n - (LSM_STEP-1)*8
 330  */
 331 #define __asm_copy_to_user_64bit_rapf_loop(to,  from, ret, n, id)\
 332         __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id,           \
 333                 "LSR    D0Ar2, D0Ar2, #8\n"                             \
 334                 "ANDS   D0Ar2, D0Ar2, #0x7\n"                           \
 335                 "ADDZ   D0Ar2, D0Ar2, #4\n"                             \
 336                 "SUB    D0Ar2, D0Ar2, #1\n"                             \
 337                 "MOV    D1Ar1, #4\n"                                    \
 338                 "SUB    D0Ar2, D1Ar1, D0Ar2\n"                          \
 339                 "LSL    D0Ar2, D0Ar2, #3\n"                             \
 340                 "LSL    D1Ar1, D1Ar1, #3\n"                             \
 341                 "SUB    D1Ar1, D1Ar1, D0Ar2\n"                          \
 342                 "SUB    %0, %0, #8\n"                                   \
 343                 "SUB    %1,     %1,D0Ar2\n"                             \
 344                 "SUB    %3, %3, D1Ar1\n")
 345
 346 /*
 347  *      optimized copying loop using RAPF when 32 bit aligned
 348  *
 349  *      n               will be automatically decremented inside the loop
 350  *      ret             will be left intact. if error occurs we will rewind
 351  *                      so that the original non optimized code will fill up
 352  *                      this value correctly.
 353  *
 354  *      on fault:
 355  *              >       n will hold total number of uncopied bytes
 356  *
 357  *              >       {'to','from'} will be rewind back so that
 358  *                      the non-optimized code will do the proper fix up
 359  *
 360  *      DCACHE drops the cacheline which helps in reducing cache
 361  *      pollution.
 362  *
 363  *      We introduce an extra SETD at the end of the loop to
 364  *      ensure we don't fall off the loop before we catch all
 365  *      erros.
 366  *
 367  *      NOTICE:
 368  *              LSM_STEP in TXSTATUS must be cleared in fix up code.
 369  *              since we're using M{S,G}ETL, a fault might happen at
 370  *              any address in the middle of M{S,G}ETL causing
 371  *              the value of LSM_STEP to be incorrect which can
 372  *              cause subsequent use of M{S,G}ET{L,D} to go wrong.
 373  *              ie: if LSM_STEP was 1 when a fault occurs, the
 374  *              next call to M{S,G}ET{L,D} will skip the first
 375  *              copy/getting as it think that the first 1 has already
 376  *              been done.
 377  *
 378  */
 379 #define __asm_copy_user_32bit_rapf_loop(                                \
 380                         to,     from, ret, n, id, FIXUP)                \
 381         asm volatile (                                                  \
 382                         ".balign 8\n"                                   \
 383                 "       MOV     RAPF, %1\n"                             \
 384                 "       MSETL   [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \
 385                 "       MOV     D0Ar6, #0\n"                            \
 386                 "       LSR     D1Ar5, %3, #6\n"                        \
 387                 "       SUB     TXRPT, D1Ar5, #2\n"                     \
 388                 "       MOV     RAPF, %1\n"                             \
 389                 "$Lloop"id":\n"                                         \
 390                 "       ADD     RAPF, %1, #64\n"                        \
 391                 "21:    MGETD   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 392                 "22:    MSETD   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 393                 "23:    SUB     %3, %3, #16\n"                          \
 394                 "24:    MGETD   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 395                 "25:    MSETD   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 396                 "26:    SUB     %3, %3, #16\n"                          \
 397                 "27:    MGETD   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 398                 "28:    MSETD   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 399                 "29:    SUB     %3, %3, #16\n"                          \
 400                 "30:    MGETD   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 401                 "31:    MSETD   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 402                 "32:    SUB     %3, %3, #16\n"                          \
 403                 "       DCACHE  [%1+#-64], D0Ar6\n"                     \
 404                 "       BR      $Lloop"id"\n"                           \
 405                                                                         \
 406                 "       MOV     RAPF, %1\n"                             \
 407                 "33:    MGETD   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 408                 "34:    MSETD   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 409                 "35:    SUB     %3, %3, #16\n"                          \
 410                 "36:    MGETD   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 411                 "37:    MSETD   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 412                 "38:    SUB     %3, %3, #16\n"                          \
 413                 "39:    MGETD   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 414                 "40:    MSETD   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 415                 "41:    SUB     %3, %3, #16\n"                          \
 416                 "42:    MGETD   D0FrT, D0.5, D0.6, D0.7, [%1++]\n"      \
 417                 "43:    MSETD   [%0++], D0FrT, D0.5, D0.6, D0.7\n"      \
 418                 "44:    SETD    [%0+#-4], D0.7\n"                       \
 419                 "       SUB     %3, %3, #16\n"                          \
 420                 "1:     DCACHE  [%1+#-64], D0Ar6\n"                     \
 421                 "       GETL    D0Ar6, D1Ar5, [A0StP+#-40]\n"           \
 422                 "       GETL    D0FrT, D1RtP, [A0StP+#-32]\n"           \
 423                 "       GETL    D0.5, D1.5, [A0StP+#-24]\n"             \
 424                 "       GETL    D0.6, D1.6, [A0StP+#-16]\n"             \
 425                 "       GETL    D0.7, D1.7, [A0StP+#-8]\n"              \
 426                 "       SUB A0StP, A0StP, #40\n"                        \
 427                 "       .section .fixup,\"ax\"\n"                       \
 428                 "3:     MOV     D0Ar2, TXSTATUS\n"                      \
 429                 "       MOV     D1Ar1, TXSTATUS\n"                      \
 430                 "       AND     D1Ar1, D1Ar1, #0xFFFFF8FF\n"            \
 431                 "       MOV     TXSTATUS, D1Ar1\n"                      \
 432                         FIXUP                                           \
 433                 "       MOVT    D0Ar2, #HI(1b)\n"                       \
 434                 "       JUMP    D0Ar2, #LO(1b)\n"                       \
 435                 "       .previous\n"                                    \
 436                 "       .section __ex_table,\"a\"\n"                    \
 437                 "       .long 21b,3b\n"                                 \
 438                 "       .long 22b,3b\n"                                 \
 439                 "       .long 23b,3b\n"                                 \
 440                 "       .long 24b,3b\n"                                 \
 441                 "       .long 25b,3b\n"                                 \
 442                 "       .long 26b,3b\n"                                 \
 443                 "       .long 27b,3b\n"                                 \
 444                 "       .long 28b,3b\n"                                 \
 445                 "       .long 29b,3b\n"                                 \
 446                 "       .long 30b,3b\n"                                 \
 447                 "       .long 31b,3b\n"                                 \
 448                 "       .long 32b,3b\n"                                 \
 449                 "       .long 33b,3b\n"                                 \
 450                 "       .long 34b,3b\n"                                 \
 451                 "       .long 35b,3b\n"                                 \
 452                 "       .long 36b,3b\n"                                 \
 453                 "       .long 37b,3b\n"                                 \
 454                 "       .long 38b,3b\n"                                 \
 455                 "       .long 39b,3b\n"                                 \
 456                 "       .long 40b,3b\n"                                 \
 457                 "       .long 41b,3b\n"                                 \
 458                 "       .long 42b,3b\n"                                 \
 459                 "       .long 43b,3b\n"                                 \
 460                 "       .long 44b,3b\n"                                 \
 461                 "       .previous\n"                                    \
 462                 : "=r" (to), "=r" (from), "=r" (ret), "=d" (n)          \
 463                 : "0" (to), "1" (from), "2" (ret), "3" (n)              \
 464                 : "D1Ar1", "D0Ar2", "cc", "memory")
 465
 466 /*      rewind 'to' and 'from'  pointers when a fault occurs
 467  *
 468  *      Rationale:
 469  *              A fault always occurs on writing to user buffer. A fault
 470  *              is at a single address, so we need to rewind by only 4
 471  *              bytes.
 472  *              Since we do a complete read from kernel buffer before
 473  *              writing, we need to rewind it also. The amount to be
 474  *              rewind equals the number of faulty writes in MSETD
 475  *              which is: [4 - (LSM_STEP-1)]*4
 476  *              LSM_STEP is bits 10:8 in TXSTATUS which is already read
 477  *              and stored in D0Ar2
 478  *
 479  *              NOTE: If a fault occurs at the last operation in M{G,S}ETL
 480  *                      LSM_STEP will be 0. ie: we do 4 writes in our case, if
 481  *                      a fault happens at the 4th write, LSM_STEP will be 0
 482  *                      instead of 4. The code copes with that.
 483  *
 484  *              n is updated by the number of successful writes, which is:
 485  *              n = n - (LSM_STEP-1)*4
 486  */
 487 #define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\
 488         __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id,           \
 489                 "LSR    D0Ar2, D0Ar2, #8\n"                             \
 490                 "ANDS   D0Ar2, D0Ar2, #0x7\n"                           \
 491                 "ADDZ   D0Ar2, D0Ar2, #4\n"                             \
 492                 "SUB    D0Ar2, D0Ar2, #1\n"                             \
 493                 "MOV    D1Ar1, #4\n"                                    \
 494                 "SUB    D0Ar2, D1Ar1, D0Ar2\n"                          \
 495                 "LSL    D0Ar2, D0Ar2, #2\n"                             \
 496                 "LSL    D1Ar1, D1Ar1, #2\n"                             \
 497                 "SUB    D1Ar1, D1Ar1, D0Ar2\n"                          \
 498                 "SUB    %0, %0, #4\n"                                   \
 499                 "SUB    %1,     %1,     D0Ar2\n"                        \
 500                 "SUB    %3, %3, D1Ar1\n")
 501
 502 unsigned long raw_copy_to_user(void __user *pdst, const void *psrc,
 503                                unsigned long n)
 504 {
 505         register char __user *dst asm ("A0.2") = pdst;
 506         register const char *src asm ("A1.2") = psrc;
 507         unsigned long retn = 0;
 508
 509         if (n == 0)
 510                 return 0;
 511
 512         if ((unsigned long) src & 1) {
 513                 __asm_copy_to_user_1(dst, src, retn);
 514                 n--;
 515                 if (retn)
 516                         return retn + n;
 517         }
 518         if ((unsigned long) dst & 1) {
 519                 /* Worst case - byte copy */
 520                 while (n > 0) {
 521                         __asm_copy_to_user_1(dst, src, retn);
 522                         n--;
 523                         if (retn)
 524                                 return retn + n;
 525                 }
 526         }
 527         if (((unsigned long) src & 2) && n >= 2) {
 528                 __asm_copy_to_user_2(dst, src, retn);
 529                 n -= 2;
 530                 if (retn)
 531                         return retn + n;
 532         }
 533         if ((unsigned long) dst & 2) {
 534                 /* Second worst case - word copy */
 535                 while (n >= 2) {
 536                         __asm_copy_to_user_2(dst, src, retn);
 537                         n -= 2;
 538                         if (retn)
 539                                 return retn + n;
 540                 }
 541         }
 542
 543 #ifdef USE_RAPF
 544         /* 64 bit copy loop */
 545         if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
 546                 if (n >= RAPF_MIN_BUF_SIZE) {
 547                         /* copy user using 64 bit rapf copy */
 548                         __asm_copy_to_user_64bit_rapf_loop(dst, src, retn,
 549                                                         n, "64cu");
 550                 }
 551                 while (n >= 8) {
 552                         __asm_copy_to_user_8x64(dst, src, retn);
 553                         n -= 8;
 554                         if (retn)
 555                                 return retn + n;
 556                 }
 557         }
 558         if (n >= RAPF_MIN_BUF_SIZE) {
 559                 /* copy user using 32 bit rapf copy */
 560                 __asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu");
 561         }
 562 #else
 563         /* 64 bit copy loop */
 564         if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) {
 565                 while (n >= 8) {
 566                         __asm_copy_to_user_8x64(dst, src, retn);
 567                         n -= 8;
 568                         if (retn)
 569                                 return retn + n;
 570                 }
 571         }
 572 #endif
 573
 574         while (n >= 16) {
 575                 __asm_copy_to_user_16(dst, src, retn);
 576                 n -= 16;
 577                 if (retn)
 578                         return retn + n;
 579         }
 580
 581         while (n >= 4) {
 582                 __asm_copy_to_user_4(dst, src, retn);
 583                 n -= 4;
 584                 if (retn)
 585                         return retn + n;
 586         }
 587
 588         switch (n) {
 589         case 0:
 590                 break;
 591         case 1:
 592                 __asm_copy_to_user_1(dst, src, retn);
 593                 break;
 594         case 2:
 595                 __asm_copy_to_user_2(dst, src, retn);
 596                 break;
 597         case 3:
 598                 __asm_copy_to_user_3(dst, src, retn);
 599                 break;
 600         }
 601
 602         /*
 603          * If we get here, retn correctly reflects the number of failing
 604          * bytes.
 605          */
 606         return retn;
 607 }
 608 EXPORT_SYMBOL(raw_copy_to_user);
 609
 610 #define __asm_copy_from_user_1(to, from, ret) \
 611         __asm_copy_user_cont(to, from, ret,     \
 612                 "       GETB D1Ar1,[%1++]\n"    \
 613                 "2:     SETB [%0++],D1Ar1\n",   \
 614                 "3:     ADD  %2,%2,#1\n",       \
 615                 "       .long 2b,3b\n")
 616
 617 #define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 618         __asm_copy_user_cont(to, from, ret,             \
 619                 "       GETW D1Ar1,[%1++]\n"            \
 620                 "2:     SETW [%0++],D1Ar1\n" COPY,      \
 621                 "3:     ADD  %2,%2,#2\n" FIXUP,         \
 622                 "       .long 2b,3b\n" TENTRY)
 623
 624 #define __asm_copy_from_user_2(to, from, ret) \
 625         __asm_copy_from_user_2x_cont(to, from, ret, "", "", "")
 626
 627 #define __asm_copy_from_user_3(to, from, ret)           \
 628         __asm_copy_from_user_2x_cont(to, from, ret,     \
 629                 "       GETB D1Ar1,[%1++]\n"            \
 630                 "4:     SETB [%0++],D1Ar1\n",           \
 631                 "5:     ADD  %2,%2,#1\n",               \
 632                 "       .long 4b,5b\n")
 633
 634 #define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \
 635         __asm_copy_user_cont(to, from, ret,             \
 636                 "       GETD D1Ar1,[%1++]\n"            \
 637                 "2:     SETD [%0++],D1Ar1\n" COPY,      \
 638                 "3:     ADD  %2,%2,#4\n" FIXUP,         \
 639                 "       .long 2b,3b\n" TENTRY)
 640
 641 #define __asm_copy_from_user_4(to, from, ret) \
 642         __asm_copy_from_user_4x_cont(to, from, ret, "", "", "")
 643
 644 #define __asm_copy_from_user_8x64(to, from, ret) \
 645         asm volatile (                          \
 646                 "       GETL D0Ar2,D1Ar1,[%1++]\n"      \
 647                 "2:     SETL [%0++],D0Ar2,D1Ar1\n"      \
 648                 "1:\n"                                  \
 649                 "       .section .fixup,\"ax\"\n"       \
 650                 "3:     ADD  %2,%2,#8\n"                \
 651                 "       MOVT    D0Ar2,#HI(1b)\n"        \
 652                 "       JUMP    D0Ar2,#LO(1b)\n"        \
 653                 "       .previous\n"                    \
 654                 "       .section __ex_table,\"a\"\n"    \
 655                 "       .long 2b,3b\n"                  \
 656                 "       .previous\n"                    \
 657                 : "=a" (to), "=r" (from), "=r" (ret)    \
 658                 : "0" (to), "1" (from), "2" (ret)       \
 659                 : "D1Ar1", "D0Ar2", "memory")
 660
 661 /*      rewind 'from' pointer when a fault occurs
 662  *
 663  *      Rationale:
 664  *              A fault occurs while reading from user buffer, which is the
 665  *              source.
 666  *              Since we don't write to kernel buffer until we read first,
 667  *              the kernel buffer is at the right state and needn't be
 668  *              corrected, but the source must be rewound to the beginning of
 669  *              the block, which is LSM_STEP*8 bytes.
 670  *              LSM_STEP is bits 10:8 in TXSTATUS which is already read
 671  *              and stored in D0Ar2
 672  *
 673  *              NOTE: If a fault occurs at the last operation in M{G,S}ETL
 674  *                      LSM_STEP will be 0. ie: we do 4 writes in our case, if
 675  *                      a fault happens at the 4th write, LSM_STEP will be 0
 676  *                      instead of 4. The code copes with that.
 677  */
 678 #define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id)      \
 679         __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id,           \
 680                 "LSR    D0Ar2, D0Ar2, #5\n"                             \
 681                 "ANDS   D0Ar2, D0Ar2, #0x38\n"                          \
 682                 "ADDZ   D0Ar2, D0Ar2, #32\n"                            \
 683                 "SUB    %1, %1, D0Ar2\n")
 684
 685 /*      rewind 'from' pointer when a fault occurs
 686  *
 687  *      Rationale:
 688  *              A fault occurs while reading from user buffer, which is the
 689  *              source.
 690  *              Since we don't write to kernel buffer until we read first,
 691  *              the kernel buffer is at the right state and needn't be
 692  *              corrected, but the source must be rewound to the beginning of
 693  *              the block, which is LSM_STEP*4 bytes.
 694  *              LSM_STEP is bits 10:8 in TXSTATUS which is already read
 695  *              and stored in D0Ar2
 696  *
 697  *              NOTE: If a fault occurs at the last operation in M{G,S}ETL
 698  *                      LSM_STEP will be 0. ie: we do 4 writes in our case, if
 699  *                      a fault happens at the 4th write, LSM_STEP will be 0
 700  *                      instead of 4. The code copes with that.
 701  */
 702 #define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id)      \
 703         __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id,           \
 704                 "LSR    D0Ar2, D0Ar2, #6\n"                             \
 705                 "ANDS   D0Ar2, D0Ar2, #0x1c\n"                          \
 706                 "ADDZ   D0Ar2, D0Ar2, #16\n"                            \
 707                 "SUB    %1, %1, D0Ar2\n")
 708
 709
 710 /*
 711  * Copy from user to kernel. The return-value is the number of bytes that were
 712  * inaccessible.
 713  */
 714 unsigned long raw_copy_from_user(void *pdst, const void __user *psrc,
 715                                  unsigned long n)
 716 {
 717         register char *dst asm ("A0.2") = pdst;
 718         register const char __user *src asm ("A1.2") = psrc;
 719         unsigned long retn = 0;
 720
 721         if (n == 0)
 722                 return 0;
 723
 724         if ((unsigned long) src & 1) {
 725                 __asm_copy_from_user_1(dst, src, retn);
 726                 n--;
 727                 if (retn)
 728                         return retn + n;
 729         }
 730         if ((unsigned long) dst & 1) {
 731                 /* Worst case - byte copy */
 732                 while (n > 0) {
 733                         __asm_copy_from_user_1(dst, src, retn);
 734                         n--;
 735                         if (retn)
 736                                 return retn + n;
 737                 }
 738         }
 739         if (((unsigned long) src & 2) && n >= 2) {
 740                 __asm_copy_from_user_2(dst, src, retn);
 741                 n -= 2;
 742                 if (retn)
 743                         return retn + n;
 744         }
 745         if ((unsigned long) dst & 2) {
 746                 /* Second worst case - word copy */
 747                 while (n >= 2) {
 748                         __asm_copy_from_user_2(dst, src, retn);
 749                         n -= 2;
 750                         if (retn)
 751                                 return retn + n;
 752                 }
 753         }
 754
 755 #ifdef USE_RAPF
 756         /* 64 bit copy loop */
 757         if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
 758                 if (n >= RAPF_MIN_BUF_SIZE) {
 759                         /* Copy using fast 64bit rapf */
 760                         __asm_copy_from_user_64bit_rapf_loop(dst, src, retn,
 761                                                         n, "64cuz");
 762                 }
 763                 while (n >= 8) {
 764                         __asm_copy_from_user_8x64(dst, src, retn);
 765                         n -= 8;
 766                         if (retn)
 767                                 return retn + n;
 768                 }
 769         }
 770
 771         if (n >= RAPF_MIN_BUF_SIZE) {
 772                 /* Copy using fast 32bit rapf */
 773                 __asm_copy_from_user_32bit_rapf_loop(dst, src, retn,
 774                                                 n, "32cuz");
 775         }
 776 #else
 777         /* 64 bit copy loop */
 778         if (!(((unsigned long) src | (unsigned long) dst) & 7)) {
 779                 while (n >= 8) {
 780                         __asm_copy_from_user_8x64(dst, src, retn);
 781                         n -= 8;
 782                         if (retn)
 783                                 return retn + n;
 784                 }
 785         }
 786 #endif
 787
 788         while (n >= 4) {
 789                 __asm_copy_from_user_4(dst, src, retn);
 790                 n -= 4;
 791
 792                 if (retn)
 793                         return retn + n;
 794         }
 795
 796         /* If we get here, there were no memory read faults.  */
 797         switch (n) {
 798                 /* These copies are at least "naturally aligned" (so we don't
 799                    have to check each byte), due to the src alignment code.
 800                    The *_3 case *will* get the correct count for retn.  */
 801         case 0:
 802                 /* This case deliberately left in (if you have doubts check the
 803                    generated assembly code).  */
 804                 break;
 805         case 1:
 806                 __asm_copy_from_user_1(dst, src, retn);
 807                 break;
 808         case 2:
 809                 __asm_copy_from_user_2(dst, src, retn);
 810                 break;
 811         case 3:
 812                 __asm_copy_from_user_3(dst, src, retn);
 813                 break;
 814         }
 815
 816         /* If we get here, retn correctly reflects the number of failing
 817            bytes.  */
 818         return retn;
 819 }
 820 EXPORT_SYMBOL(raw_copy_from_user);
 821
 822 #define __asm_clear_8x64(to, ret) \
 823         asm volatile (                                  \
 824                 "       MOV  D0Ar2,#0\n"                \
 825                 "       MOV  D1Ar1,#0\n"                \
 826                 "       SETL [%0],D0Ar2,D1Ar1\n"        \
 827                 "2:     SETL [%0++],D0Ar2,D1Ar1\n"      \
 828                 "1:\n"                                  \
 829                 "       .section .fixup,\"ax\"\n"       \
 830                 "3:     ADD  %1,%1,#8\n"                \
 831                 "       MOVT    D0Ar2,#HI(1b)\n"        \
 832                 "       JUMP    D0Ar2,#LO(1b)\n"        \
 833                 "       .previous\n"                    \
 834                 "       .section __ex_table,\"a\"\n"    \
 835                 "       .long 2b,3b\n"                  \
 836                 "       .previous\n"                    \
 837                 : "=r" (to), "=r" (ret) \
 838                 : "0" (to), "1" (ret)   \
 839                 : "D1Ar1", "D0Ar2", "memory")
 840
 841 /* Zero userspace.  */
 842
 843 #define __asm_clear(to, ret, CLEAR, FIXUP, TENTRY) \
 844         asm volatile (                                  \
 845                 "       MOV D1Ar1,#0\n"                 \
 846                         CLEAR                           \
 847                 "1:\n"                                  \
 848                 "       .section .fixup,\"ax\"\n"       \
 849                         FIXUP                           \
 850                 "       MOVT    D1Ar1,#HI(1b)\n"        \
 851                 "       JUMP    D1Ar1,#LO(1b)\n"        \
 852                 "       .previous\n"                    \
 853                 "       .section __ex_table,\"a\"\n"    \
 854                         TENTRY                          \
 855                 "       .previous"                      \
 856                 : "=r" (to), "=r" (ret)                 \
 857                 : "0" (to), "1" (ret)                   \
 858                 : "D1Ar1", "memory")
 859
 860 #define __asm_clear_1(to, ret) \
 861         __asm_clear(to, ret,                    \
 862                 "       SETB [%0],D1Ar1\n"      \
 863                 "2:     SETB [%0++],D1Ar1\n",   \
 864                 "3:     ADD  %1,%1,#1\n",       \
 865                 "       .long 2b,3b\n")
 866
 867 #define __asm_clear_2(to, ret) \
 868         __asm_clear(to, ret,                    \
 869                 "       SETW [%0],D1Ar1\n"      \
 870                 "2:     SETW [%0++],D1Ar1\n",   \
 871                 "3:     ADD  %1,%1,#2\n",       \
 872                 "       .long 2b,3b\n")
 873
 874 #define __asm_clear_3(to, ret) \
 875         __asm_clear(to, ret,                    \
 876                  "2:    SETW [%0++],D1Ar1\n"    \
 877                  "      SETB [%0],D1Ar1\n"      \
 878                  "3:    SETB [%0++],D1Ar1\n",   \
 879                  "4:    ADD  %1,%1,#2\n"        \
 880                  "5:    ADD  %1,%1,#1\n",       \
 881                  "      .long 2b,4b\n"          \
 882                  "      .long 3b,5b\n")
 883
 884 #define __asm_clear_4x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
 885         __asm_clear(to, ret,                            \
 886                 "       SETD [%0],D1Ar1\n"              \
 887                 "2:     SETD [%0++],D1Ar1\n" CLEAR,     \
 888                 "3:     ADD  %1,%1,#4\n" FIXUP,         \
 889                 "       .long 2b,3b\n" TENTRY)
 890
 891 #define __asm_clear_4(to, ret) \
 892         __asm_clear_4x_cont(to, ret, "", "", "")
 893
 894 #define __asm_clear_8x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
 895         __asm_clear_4x_cont(to, ret,                    \
 896                 "       SETD [%0],D1Ar1\n"              \
 897                 "4:     SETD [%0++],D1Ar1\n" CLEAR,     \
 898                 "5:     ADD  %1,%1,#4\n" FIXUP,         \
 899                 "       .long 4b,5b\n" TENTRY)
 900
 901 #define __asm_clear_8(to, ret) \
 902         __asm_clear_8x_cont(to, ret, "", "", "")
 903
 904 #define __asm_clear_12x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
 905         __asm_clear_8x_cont(to, ret,                    \
 906                 "       SETD [%0],D1Ar1\n"              \
 907                 "6:     SETD [%0++],D1Ar1\n" CLEAR,     \
 908                 "7:     ADD  %1,%1,#4\n" FIXUP,         \
 909                 "       .long 6b,7b\n" TENTRY)
 910
 911 #define __asm_clear_12(to, ret) \
 912         __asm_clear_12x_cont(to, ret, "", "", "")
 913
 914 #define __asm_clear_16x_cont(to, ret, CLEAR, FIXUP, TENTRY) \
 915         __asm_clear_12x_cont(to, ret,                   \
 916                 "       SETD [%0],D1Ar1\n"              \
 917                 "8:     SETD [%0++],D1Ar1\n" CLEAR,     \
 918                 "9:     ADD  %1,%1,#4\n" FIXUP,         \
 919                 "       .long 8b,9b\n" TENTRY)
 920
 921 #define __asm_clear_16(to, ret) \
 922         __asm_clear_16x_cont(to, ret, "", "", "")
 923
 924 unsigned long __do_clear_user(void __user *pto, unsigned long pn)
 925 {
 926         register char __user *dst asm ("D0Re0") = pto;
 927         register unsigned long n asm ("D1Re0") = pn;
 928         register unsigned long retn asm ("D0Ar6") = 0;
 929
 930         if ((unsigned long) dst & 1) {
 931                 __asm_clear_1(dst, retn);
 932                 n--;
 933         }
 934
 935         if ((unsigned long) dst & 2) {
 936                 __asm_clear_2(dst, retn);
 937                 n -= 2;
 938         }
 939
 940         /* 64 bit copy loop */
 941         if (!((__force unsigned long) dst & 7)) {
 942                 while (n >= 8) {
 943                         __asm_clear_8x64(dst, retn);
 944                         n -= 8;
 945                 }
 946         }
 947
 948         while (n >= 16) {
 949                 __asm_clear_16(dst, retn);
 950                 n -= 16;
 951         }
 952
 953         while (n >= 4) {
 954                 __asm_clear_4(dst, retn);
 955                 n -= 4;
 956         }
 957
 958         switch (n) {
 959         case 0:
 960                 break;
 961         case 1:
 962                 __asm_clear_1(dst, retn);
 963                 break;
 964         case 2:
 965                 __asm_clear_2(dst, retn);
 966                 break;
 967         case 3:
 968                 __asm_clear_3(dst, retn);
 969                 break;
 970         }
 971
 972         return retn;
 973 }
 974 EXPORT_SYMBOL(__do_clear_user);
 975
 976 unsigned char __get_user_asm_b(const void __user *addr, long *err)
 977 {
 978         register unsigned char x asm ("D0Re0") = 0;
 979         asm volatile (
 980                 "       GETB %0,[%2]\n"
 981                 "1:\n"
 982                 "       GETB %0,[%2]\n"
 983                 "2:\n"
 984                 "       .section .fixup,\"ax\"\n"
 985                 "3:     MOV     D0FrT,%3\n"
 986                 "       SETD    [%1],D0FrT\n"
 987                 "       MOVT    D0FrT,#HI(2b)\n"
 988                 "       JUMP    D0FrT,#LO(2b)\n"
 989                 "       .previous\n"
 990                 "       .section __ex_table,\"a\"\n"
 991                 "       .long 1b,3b\n"
 992                 "       .previous\n"
 993                 : "=r" (x)
 994                 : "r" (err), "r" (addr), "P" (-EFAULT)
 995                 : "D0FrT");
 996         return x;
 997 }
 998 EXPORT_SYMBOL(__get_user_asm_b);
 999
1000 unsigned short __get_user_asm_w(const void __user *addr, long *err)
1001 {
1002         register unsigned short x asm ("D0Re0") = 0;
1003         asm volatile (
1004                 "       GETW %0,[%2]\n"
1005                 "1:\n"
1006                 "       GETW %0,[%2]\n"
1007                 "2:\n"
1008                 "       .section .fixup,\"ax\"\n"
1009                 "3:     MOV     D0FrT,%3\n"
1010                 "       SETD    [%1],D0FrT\n"
1011                 "       MOVT    D0FrT,#HI(2b)\n"
1012                 "       JUMP    D0FrT,#LO(2b)\n"
1013                 "       .previous\n"
1014                 "       .section __ex_table,\"a\"\n"
1015                 "       .long 1b,3b\n"
1016                 "       .previous\n"
1017                 : "=r" (x)
1018                 : "r" (err), "r" (addr), "P" (-EFAULT)
1019                 : "D0FrT");
1020         return x;
1021 }
1022 EXPORT_SYMBOL(__get_user_asm_w);
1023
1024 unsigned int __get_user_asm_d(const void __user *addr, long *err)
1025 {
1026         register unsigned int x asm ("D0Re0") = 0;
1027         asm volatile (
1028                 "       GETD %0,[%2]\n"
1029                 "1:\n"
1030                 "       GETD %0,[%2]\n"
1031                 "2:\n"
1032                 "       .section .fixup,\"ax\"\n"
1033                 "3:     MOV     D0FrT,%3\n"
1034                 "       SETD    [%1],D0FrT\n"
1035                 "       MOVT    D0FrT,#HI(2b)\n"
1036                 "       JUMP    D0FrT,#LO(2b)\n"
1037                 "       .previous\n"
1038                 "       .section __ex_table,\"a\"\n"
1039                 "       .long 1b,3b\n"
1040                 "       .previous\n"
1041                 : "=r" (x)
1042                 : "r" (err), "r" (addr), "P" (-EFAULT)
1043                 : "D0FrT");
1044         return x;
1045 }
1046 EXPORT_SYMBOL(__get_user_asm_d);
1047
1048 unsigned long long __get_user_asm_l(const void __user *addr, long *err)
1049 {
1050         register unsigned long long x asm ("D0Re0") = 0;
1051         asm volatile (
1052                 "       GETL %0,%t0,[%2]\n"
1053                 "1:\n"
1054                 "       GETL %0,%t0,[%2]\n"
1055                 "2:\n"
1056                 "       .section .fixup,\"ax\"\n"
1057                 "3:     MOV     D0FrT,%3\n"
1058                 "       SETD    [%1],D0FrT\n"
1059                 "       MOVT    D0FrT,#HI(2b)\n"
1060                 "       JUMP    D0FrT,#LO(2b)\n"
1061                 "       .previous\n"
1062                 "       .section __ex_table,\"a\"\n"
1063                 "       .long 1b,3b\n"
1064                 "       .previous\n"
1065                 : "=r" (x)
1066                 : "r" (err), "r" (addr), "P" (-EFAULT)
1067                 : "D0FrT");
1068         return x;
1069 }
1070 EXPORT_SYMBOL(__get_user_asm_l);
1071
1072 long __put_user_asm_b(unsigned int x, void __user *addr)
1073 {
1074         register unsigned int err asm ("D0Re0") = 0;
1075         asm volatile (
1076                 "       MOV  %0,#0\n"
1077                 "       SETB [%2],%1\n"
1078                 "1:\n"
1079                 "       SETB [%2],%1\n"
1080                 "2:\n"
1081                 ".section .fixup,\"ax\"\n"
1082                 "3:     MOV     %0,%3\n"
1083                 "       MOVT    D0FrT,#HI(2b)\n"
1084                 "       JUMP    D0FrT,#LO(2b)\n"
1085                 ".previous\n"
1086                 ".section __ex_table,\"a\"\n"
1087                 "       .long 1b,3b\n"
1088                 ".previous"
1089                 : "=r"(err)
1090                 : "d" (x), "a" (addr), "P"(-EFAULT)
1091                 : "D0FrT");
1092         return err;
1093 }
1094 EXPORT_SYMBOL(__put_user_asm_b);
1095
1096 long __put_user_asm_w(unsigned int x, void __user *addr)
1097 {
1098         register unsigned int err asm ("D0Re0") = 0;
1099         asm volatile (
1100                 "       MOV  %0,#0\n"
1101                 "       SETW [%2],%1\n"
1102                 "1:\n"
1103                 "       SETW [%2],%1\n"
1104                 "2:\n"
1105                 ".section .fixup,\"ax\"\n"
1106                 "3:     MOV     %0,%3\n"
1107                 "       MOVT    D0FrT,#HI(2b)\n"
1108                 "       JUMP    D0FrT,#LO(2b)\n"
1109                 ".previous\n"
1110                 ".section __ex_table,\"a\"\n"
1111                 "       .long 1b,3b\n"
1112                 ".previous"
1113                 : "=r"(err)
1114                 : "d" (x), "a" (addr), "P"(-EFAULT)
1115                 : "D0FrT");
1116         return err;
1117 }
1118 EXPORT_SYMBOL(__put_user_asm_w);
1119
1120 long __put_user_asm_d(unsigned int x, void __user *addr)
1121 {
1122         register unsigned int err asm ("D0Re0") = 0;
1123         asm volatile (
1124                 "       MOV  %0,#0\n"
1125                 "       SETD [%2],%1\n"
1126                 "1:\n"
1127                 "       SETD [%2],%1\n"
1128                 "2:\n"
1129                 ".section .fixup,\"ax\"\n"
1130                 "3:     MOV     %0,%3\n"
1131                 "       MOVT    D0FrT,#HI(2b)\n"
1132                 "       JUMP    D0FrT,#LO(2b)\n"
1133                 ".previous\n"
1134                 ".section __ex_table,\"a\"\n"
1135                 "       .long 1b,3b\n"
1136                 ".previous"
1137                 : "=r"(err)
1138                 : "d" (x), "a" (addr), "P"(-EFAULT)
1139                 : "D0FrT");
1140         return err;
1141 }
1142 EXPORT_SYMBOL(__put_user_asm_d);
1143
1144 long __put_user_asm_l(unsigned long long x, void __user *addr)
1145 {
1146         register unsigned int err asm ("D0Re0") = 0;
1147         asm volatile (
1148                 "       MOV  %0,#0\n"
1149                 "       SETL [%2],%1,%t1\n"
1150                 "1:\n"
1151                 "       SETL [%2],%1,%t1\n"
1152                 "2:\n"
1153                 ".section .fixup,\"ax\"\n"
1154                 "3:     MOV     %0,%3\n"
1155                 "       MOVT    D0FrT,#HI(2b)\n"
1156                 "       JUMP    D0FrT,#LO(2b)\n"
1157                 ".previous\n"
1158                 ".section __ex_table,\"a\"\n"
1159                 "       .long 1b,3b\n"
1160                 ".previous"
1161                 : "=r"(err)
1162                 : "d" (x), "a" (addr), "P"(-EFAULT)
1163                 : "D0FrT");
1164         return err;
1165 }
1166 EXPORT_SYMBOL(__put_user_asm_l);
1167
1168 long strnlen_user(const char __user *src, long count)
1169 {
1170         long res;
1171
1172         if (!access_ok(VERIFY_READ, src, 0))
1173                 return 0;
1174
1175         asm volatile (" MOV     D0Ar4, %1\n"
1176                       " MOV     D0Ar6, %2\n"
1177                       "0:\n"
1178                       " SUBS    D0FrT, D0Ar6, #0\n"
1179                       " SUB     D0Ar6, D0Ar6, #1\n"
1180                       " BLE     2f\n"
1181                       " GETB    D0FrT, [D0Ar4+#1++]\n"
1182                       "1:\n"
1183                       " TST     D0FrT, #255\n"
1184                       " BNE     0b\n"
1185                       "2:\n"
1186                       " SUB     %0, %2, D0Ar6\n"
1187                       "3:\n"
1188                       " .section .fixup,\"ax\"\n"
1189                       "4:\n"
1190                       " MOV     %0, #0\n"
1191                       " MOVT    D0FrT,#HI(3b)\n"
1192                       " JUMP    D0FrT,#LO(3b)\n"
1193                       " .previous\n"
1194                       " .section __ex_table,\"a\"\n"
1195                       " .long 1b,4b\n"
1196                       " .previous\n"
1197                       : "=r" (res)
1198                       : "r" (src), "r" (count)
1199                       : "D0FrT", "D0Ar4", "D0Ar6", "cc");
1200
1201         return res;
1202 }
1203 EXPORT_SYMBOL(strnlen_user);
1204
1205 long __strncpy_from_user(char *dst, const char __user *src, long count)
1206 {
1207         long res;
1208
1209         if (count == 0)
1210                 return 0;
1211
1212         /*
1213          * Currently, in 2.4.0-test9, most ports use a simple byte-copy loop.
1214          *  So do we.
1215          *
1216          *  This code is deduced from:
1217          *
1218          *      char tmp2;
1219          *      long tmp1, tmp3;
1220          *      tmp1 = count;
1221          *      while ((*dst++ = (tmp2 = *src++)) != 0
1222          *             && --tmp1)
1223          *        ;
1224          *
1225          *      res = count - tmp1;
1226          *
1227          *  with tweaks.
1228          */
1229
1230         asm volatile (" MOV  %0,%3\n"
1231                       "1:\n"
1232                       " GETB D0FrT,[%2++]\n"
1233                       "2:\n"
1234                       " CMP  D0FrT,#0\n"
1235                       " SETB [%1++],D0FrT\n"
1236                       " BEQ  3f\n"
1237                       " SUBS %0,%0,#1\n"
1238                       " BNZ  1b\n"
1239                       "3:\n"
1240                       " SUB  %0,%3,%0\n"
1241                       "4:\n"
1242                       " .section .fixup,\"ax\"\n"
1243                       "5:\n"
1244                       " MOV  %0,%7\n"
1245                       " MOVT    D0FrT,#HI(4b)\n"
1246                       " JUMP    D0FrT,#LO(4b)\n"
1247                       " .previous\n"
1248                       " .section __ex_table,\"a\"\n"
1249                       " .long 2b,5b\n"
1250                       " .previous"
1251                       : "=r" (res), "=r" (dst), "=r" (src), "=r" (count)
1252                       : "3" (count), "1" (dst), "2" (src), "P" (-EFAULT)
1253                       : "D0FrT", "memory", "cc");
1254
1255         return res;
1256 }
1257 EXPORT_SYMBOL(__strncpy_from_user);