arch/x86/crypto/camellia_aesni_avx2_glue.c

   1 /*
   2  * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
   3  *
   4  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  */
  12
  13 #include <asm/crypto/camellia.h>
  14 #include <asm/crypto/glue_helper.h>
  15 #include <crypto/algapi.h>
  16 #include <crypto/internal/simd.h>
  17 #include <crypto/xts.h>
  18 #include <linux/crypto.h>
  19 #include <linux/err.h>
  20 #include <linux/module.h>
  21 #include <linux/types.h>
  22
  23 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
  24 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32
  25
  26 /* 32-way AVX2/AES-NI parallel cipher functions */
  27 asmlinkage void camellia_ecb_enc_32way(struct camellia_ctx *ctx, u8 *dst,
  28                                        const u8 *src);
  29 asmlinkage void camellia_ecb_dec_32way(struct camellia_ctx *ctx, u8 *dst,
  30                                        const u8 *src);
  31
  32 asmlinkage void camellia_cbc_dec_32way(struct camellia_ctx *ctx, u8 *dst,
  33                                        const u8 *src);
  34 asmlinkage void camellia_ctr_32way(struct camellia_ctx *ctx, u8 *dst,
  35                                    const u8 *src, le128 *iv);
  36
  37 asmlinkage void camellia_xts_enc_32way(struct camellia_ctx *ctx, u8 *dst,
  38                                        const u8 *src, le128 *iv);
  39 asmlinkage void camellia_xts_dec_32way(struct camellia_ctx *ctx, u8 *dst,
  40                                        const u8 *src, le128 *iv);
  41
  42 static const struct common_glue_ctx camellia_enc = {
  43         .num_funcs = 4,
  44         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  45
  46         .funcs = { {
  47                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  48                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_32way) }
  49         }, {
  50                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  51                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
  52         }, {
  53                 .num_blocks = 2,
  54                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
  55         }, {
  56                 .num_blocks = 1,
  57                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
  58         } }
  59 };
  60
  61 static const struct common_glue_ctx camellia_ctr = {
  62         .num_funcs = 4,
  63         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  64
  65         .funcs = { {
  66                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  67                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_32way) }
  68         }, {
  69                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  70                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
  71         }, {
  72                 .num_blocks = 2,
  73                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
  74         }, {
  75                 .num_blocks = 1,
  76                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
  77         } }
  78 };
  79
  80 static const struct common_glue_ctx camellia_enc_xts = {
  81         .num_funcs = 3,
  82         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  83
  84         .funcs = { {
  85                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  86                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_32way) }
  87         }, {
  88                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  89                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) }
  90         }, {
  91                 .num_blocks = 1,
  92                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) }
  93         } }
  94 };
  95
  96 static const struct common_glue_ctx camellia_dec = {
  97         .num_funcs = 4,
  98         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  99
 100         .funcs = { {
 101                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 102                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_32way) }
 103         }, {
 104                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 105                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
 106         }, {
 107                 .num_blocks = 2,
 108                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
 109         }, {
 110                 .num_blocks = 1,
 111                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
 112         } }
 113 };
 114
 115 static const struct common_glue_ctx camellia_dec_cbc = {
 116         .num_funcs = 4,
 117         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 118
 119         .funcs = { {
 120                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 121                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_32way) }
 122         }, {
 123                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 124                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
 125         }, {
 126                 .num_blocks = 2,
 127                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
 128         }, {
 129                 .num_blocks = 1,
 130                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
 131         } }
 132 };
 133
 134 static const struct common_glue_ctx camellia_dec_xts = {
 135         .num_funcs = 3,
 136         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 137
 138         .funcs = { {
 139                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 140                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_32way) }
 141         }, {
 142                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 143                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) }
 144         }, {
 145                 .num_blocks = 1,
 146                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) }
 147         } }
 148 };
 149
 150 static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
 151                            unsigned int keylen)
 152 {
 153         return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen,
 154                                  &tfm->base.crt_flags);
 155 }
 156
 157 static int ecb_encrypt(struct skcipher_request *req)
 158 {
 159         return glue_ecb_req_128bit(&camellia_enc, req);
 160 }
 161
 162 static int ecb_decrypt(struct skcipher_request *req)
 163 {
 164         return glue_ecb_req_128bit(&camellia_dec, req);
 165 }
 166
 167 static int cbc_encrypt(struct skcipher_request *req)
 168 {
 169         return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(camellia_enc_blk),
 170                                            req);
 171 }
 172
 173 static int cbc_decrypt(struct skcipher_request *req)
 174 {
 175         return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
 176 }
 177
 178 static int ctr_crypt(struct skcipher_request *req)
 179 {
 180         return glue_ctr_req_128bit(&camellia_ctr, req);
 181 }
 182
 183 static int xts_encrypt(struct skcipher_request *req)
 184 {
 185         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 186         struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 187
 188         return glue_xts_req_128bit(&camellia_enc_xts, req,
 189                                    XTS_TWEAK_CAST(camellia_enc_blk),
 190                                    &ctx->tweak_ctx, &ctx->crypt_ctx);
 191 }
 192
 193 static int xts_decrypt(struct skcipher_request *req)
 194 {
 195         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
 196         struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
 197
 198         return glue_xts_req_128bit(&camellia_dec_xts, req,
 199                                    XTS_TWEAK_CAST(camellia_enc_blk),
 200                                    &ctx->tweak_ctx, &ctx->crypt_ctx);
 201 }
 202
 203 static struct skcipher_alg camellia_algs[] = {
 204         {
 205                 .base.cra_name          = "__ecb(camellia)",
 206                 .base.cra_driver_name   = "__ecb-camellia-aesni-avx2",
 207                 .base.cra_priority      = 500,
 208                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 209                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 210                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 211                 .base.cra_module        = THIS_MODULE,
 212                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 213                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 214                 .setkey                 = camellia_setkey,
 215                 .encrypt                = ecb_encrypt,
 216                 .decrypt                = ecb_decrypt,
 217         }, {
 218                 .base.cra_name          = "__cbc(camellia)",
 219                 .base.cra_driver_name   = "__cbc-camellia-aesni-avx2",
 220                 .base.cra_priority      = 500,
 221                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 222                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 223                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 224                 .base.cra_module        = THIS_MODULE,
 225                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 226                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 227                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 228                 .setkey                 = camellia_setkey,
 229                 .encrypt                = cbc_encrypt,
 230                 .decrypt                = cbc_decrypt,
 231         }, {
 232                 .base.cra_name          = "__ctr(camellia)",
 233                 .base.cra_driver_name   = "__ctr-camellia-aesni-avx2",
 234                 .base.cra_priority      = 500,
 235                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 236                 .base.cra_blocksize     = 1,
 237                 .base.cra_ctxsize       = sizeof(struct camellia_ctx),
 238                 .base.cra_module        = THIS_MODULE,
 239                 .min_keysize            = CAMELLIA_MIN_KEY_SIZE,
 240                 .max_keysize            = CAMELLIA_MAX_KEY_SIZE,
 241                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 242                 .chunksize              = CAMELLIA_BLOCK_SIZE,
 243                 .setkey                 = camellia_setkey,
 244                 .encrypt                = ctr_crypt,
 245                 .decrypt                = ctr_crypt,
 246         }, {
 247                 .base.cra_name          = "__xts(camellia)",
 248                 .base.cra_driver_name   = "__xts-camellia-aesni-avx2",
 249                 .base.cra_priority      = 500,
 250                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
 251                 .base.cra_blocksize     = CAMELLIA_BLOCK_SIZE,
 252                 .base.cra_ctxsize       = sizeof(struct camellia_xts_ctx),
 253                 .base.cra_module        = THIS_MODULE,
 254                 .min_keysize            = 2 * CAMELLIA_MIN_KEY_SIZE,
 255                 .max_keysize            = 2 * CAMELLIA_MAX_KEY_SIZE,
 256                 .ivsize                 = CAMELLIA_BLOCK_SIZE,
 257                 .setkey                 = xts_camellia_setkey,
 258                 .encrypt                = xts_encrypt,
 259                 .decrypt                = xts_decrypt,
 260         },
 261 };
 262
 263 static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
 264
 265 static int __init camellia_aesni_init(void)
 266 {
 267         const char *feature_name;
 268
 269         if (!boot_cpu_has(X86_FEATURE_AVX) ||
 270             !boot_cpu_has(X86_FEATURE_AVX2) ||
 271             !boot_cpu_has(X86_FEATURE_AES) ||
 272             !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
 273                 pr_info("AVX2 or AES-NI instructions are not detected.\n");
 274                 return -ENODEV;
 275         }
 276
 277         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 278                                 &feature_name)) {
 279                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 280                 return -ENODEV;
 281         }
 282
 283         return simd_register_skciphers_compat(camellia_algs,
 284                                               ARRAY_SIZE(camellia_algs),
 285                                               camellia_simd_algs);
 286 }
 287
 288 static void __exit camellia_aesni_fini(void)
 289 {
 290         simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
 291                                   camellia_simd_algs);
 292 }
 293
 294 module_init(camellia_aesni_init);
 295 module_exit(camellia_aesni_fini);
 296
 297 MODULE_LICENSE("GPL");
 298 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
 299 MODULE_ALIAS_CRYPTO("camellia");
 300 MODULE_ALIAS_CRYPTO("camellia-asm");