GNU Linux-libre 4.14.266-gnu1
[releases.git] / arch / x86 / crypto / twofish_avx_glue.c
1 /*
2  * Glue Code for AVX assembler version of Twofish Cipher
3  *
4  * Copyright (C) 2012 Johannes Goetzfried
5  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6  *
7  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
22  * USA
23  *
24  */
25
26 #include <linux/module.h>
27 #include <linux/hardirq.h>
28 #include <linux/types.h>
29 #include <linux/crypto.h>
30 #include <linux/err.h>
31 #include <crypto/ablk_helper.h>
32 #include <crypto/algapi.h>
33 #include <crypto/twofish.h>
34 #include <crypto/cryptd.h>
35 #include <crypto/b128ops.h>
36 #include <crypto/ctr.h>
37 #include <crypto/lrw.h>
38 #include <crypto/xts.h>
39 #include <asm/fpu/api.h>
40 #include <asm/crypto/twofish.h>
41 #include <asm/crypto/glue_helper.h>
42 #include <crypto/scatterwalk.h>
43 #include <linux/workqueue.h>
44 #include <linux/spinlock.h>
45
46 #define TWOFISH_PARALLEL_BLOCKS 8
47
48 /* 8-way parallel cipher functions */
49 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
50                                      const u8 *src);
51 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
52                                      const u8 *src);
53
54 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
55                                      const u8 *src);
56 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
57                                  const u8 *src, le128 *iv);
58
59 asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
60                                      const u8 *src, le128 *iv);
61 asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
62                                      const u8 *src, le128 *iv);
63
64 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
65                                         const u8 *src)
66 {
67         __twofish_enc_blk_3way(ctx, dst, src, false);
68 }
69
70 static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
71 {
72         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
73                                   GLUE_FUNC_CAST(twofish_enc_blk));
74 }
75
76 static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
77 {
78         glue_xts_crypt_128bit_one(ctx, dst, src, iv,
79                                   GLUE_FUNC_CAST(twofish_dec_blk));
80 }
81
82
83 static const struct common_glue_ctx twofish_enc = {
84         .num_funcs = 3,
85         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
86
87         .funcs = { {
88                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
89                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
90         }, {
91                 .num_blocks = 3,
92                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
93         }, {
94                 .num_blocks = 1,
95                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
96         } }
97 };
98
99 static const struct common_glue_ctx twofish_ctr = {
100         .num_funcs = 3,
101         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
102
103         .funcs = { {
104                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
105                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
106         }, {
107                 .num_blocks = 3,
108                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
109         }, {
110                 .num_blocks = 1,
111                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
112         } }
113 };
114
115 static const struct common_glue_ctx twofish_enc_xts = {
116         .num_funcs = 2,
117         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
118
119         .funcs = { {
120                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
121                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
122         }, {
123                 .num_blocks = 1,
124                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
125         } }
126 };
127
128 static const struct common_glue_ctx twofish_dec = {
129         .num_funcs = 3,
130         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
131
132         .funcs = { {
133                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
134                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
135         }, {
136                 .num_blocks = 3,
137                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
138         }, {
139                 .num_blocks = 1,
140                 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
141         } }
142 };
143
144 static const struct common_glue_ctx twofish_dec_cbc = {
145         .num_funcs = 3,
146         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
147
148         .funcs = { {
149                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
150                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
151         }, {
152                 .num_blocks = 3,
153                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
154         }, {
155                 .num_blocks = 1,
156                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
157         } }
158 };
159
160 static const struct common_glue_ctx twofish_dec_xts = {
161         .num_funcs = 2,
162         .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
163
164         .funcs = { {
165                 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
166                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
167         }, {
168                 .num_blocks = 1,
169                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
170         } }
171 };
172
173 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
174                        struct scatterlist *src, unsigned int nbytes)
175 {
176         return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
177 }
178
179 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
180                        struct scatterlist *src, unsigned int nbytes)
181 {
182         return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
183 }
184
185 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
186                        struct scatterlist *src, unsigned int nbytes)
187 {
188         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
189                                        dst, src, nbytes);
190 }
191
192 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
193                        struct scatterlist *src, unsigned int nbytes)
194 {
195         return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
196                                        nbytes);
197 }
198
199 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
200                      struct scatterlist *src, unsigned int nbytes)
201 {
202         return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
203 }
204
205 static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
206 {
207         return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
208                               fpu_enabled, nbytes);
209 }
210
211 static inline void twofish_fpu_end(bool fpu_enabled)
212 {
213         glue_fpu_end(fpu_enabled);
214 }
215
216 struct crypt_priv {
217         struct twofish_ctx *ctx;
218         bool fpu_enabled;
219 };
220
221 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
222 {
223         const unsigned int bsize = TF_BLOCK_SIZE;
224         struct crypt_priv *ctx = priv;
225         int i;
226
227         ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
228
229         if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
230                 twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
231                 return;
232         }
233
234         for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
235                 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
236
237         nbytes %= bsize * 3;
238
239         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
240                 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
241 }
242
243 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
244 {
245         const unsigned int bsize = TF_BLOCK_SIZE;
246         struct crypt_priv *ctx = priv;
247         int i;
248
249         ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
250
251         if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
252                 twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
253                 return;
254         }
255
256         for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
257                 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
258
259         nbytes %= bsize * 3;
260
261         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
262                 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
263 }
264
265 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
266                        struct scatterlist *src, unsigned int nbytes)
267 {
268         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
269         be128 buf[TWOFISH_PARALLEL_BLOCKS];
270         struct crypt_priv crypt_ctx = {
271                 .ctx = &ctx->twofish_ctx,
272                 .fpu_enabled = false,
273         };
274         struct lrw_crypt_req req = {
275                 .tbuf = buf,
276                 .tbuflen = sizeof(buf),
277
278                 .table_ctx = &ctx->lrw_table,
279                 .crypt_ctx = &crypt_ctx,
280                 .crypt_fn = encrypt_callback,
281         };
282         int ret;
283
284         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
285         ret = lrw_crypt(desc, dst, src, nbytes, &req);
286         twofish_fpu_end(crypt_ctx.fpu_enabled);
287
288         return ret;
289 }
290
291 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
292                        struct scatterlist *src, unsigned int nbytes)
293 {
294         struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
295         be128 buf[TWOFISH_PARALLEL_BLOCKS];
296         struct crypt_priv crypt_ctx = {
297                 .ctx = &ctx->twofish_ctx,
298                 .fpu_enabled = false,
299         };
300         struct lrw_crypt_req req = {
301                 .tbuf = buf,
302                 .tbuflen = sizeof(buf),
303
304                 .table_ctx = &ctx->lrw_table,
305                 .crypt_ctx = &crypt_ctx,
306                 .crypt_fn = decrypt_callback,
307         };
308         int ret;
309
310         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
311         ret = lrw_crypt(desc, dst, src, nbytes, &req);
312         twofish_fpu_end(crypt_ctx.fpu_enabled);
313
314         return ret;
315 }
316
317 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
318                        struct scatterlist *src, unsigned int nbytes)
319 {
320         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
321
322         return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
323                                      XTS_TWEAK_CAST(twofish_enc_blk),
324                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
325 }
326
327 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
328                        struct scatterlist *src, unsigned int nbytes)
329 {
330         struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
331
332         return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
333                                      XTS_TWEAK_CAST(twofish_enc_blk),
334                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
335 }
336
337 static struct crypto_alg twofish_algs[10] = { {
338         .cra_name               = "__ecb-twofish-avx",
339         .cra_driver_name        = "__driver-ecb-twofish-avx",
340         .cra_priority           = 0,
341         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
342                                   CRYPTO_ALG_INTERNAL,
343         .cra_blocksize          = TF_BLOCK_SIZE,
344         .cra_ctxsize            = sizeof(struct twofish_ctx),
345         .cra_alignmask          = 0,
346         .cra_type               = &crypto_blkcipher_type,
347         .cra_module             = THIS_MODULE,
348         .cra_u = {
349                 .blkcipher = {
350                         .min_keysize    = TF_MIN_KEY_SIZE,
351                         .max_keysize    = TF_MAX_KEY_SIZE,
352                         .setkey         = twofish_setkey,
353                         .encrypt        = ecb_encrypt,
354                         .decrypt        = ecb_decrypt,
355                 },
356         },
357 }, {
358         .cra_name               = "__cbc-twofish-avx",
359         .cra_driver_name        = "__driver-cbc-twofish-avx",
360         .cra_priority           = 0,
361         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
362                                   CRYPTO_ALG_INTERNAL,
363         .cra_blocksize          = TF_BLOCK_SIZE,
364         .cra_ctxsize            = sizeof(struct twofish_ctx),
365         .cra_alignmask          = 0,
366         .cra_type               = &crypto_blkcipher_type,
367         .cra_module             = THIS_MODULE,
368         .cra_u = {
369                 .blkcipher = {
370                         .min_keysize    = TF_MIN_KEY_SIZE,
371                         .max_keysize    = TF_MAX_KEY_SIZE,
372                         .setkey         = twofish_setkey,
373                         .encrypt        = cbc_encrypt,
374                         .decrypt        = cbc_decrypt,
375                 },
376         },
377 }, {
378         .cra_name               = "__ctr-twofish-avx",
379         .cra_driver_name        = "__driver-ctr-twofish-avx",
380         .cra_priority           = 0,
381         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
382                                   CRYPTO_ALG_INTERNAL,
383         .cra_blocksize          = 1,
384         .cra_ctxsize            = sizeof(struct twofish_ctx),
385         .cra_alignmask          = 0,
386         .cra_type               = &crypto_blkcipher_type,
387         .cra_module             = THIS_MODULE,
388         .cra_u = {
389                 .blkcipher = {
390                         .min_keysize    = TF_MIN_KEY_SIZE,
391                         .max_keysize    = TF_MAX_KEY_SIZE,
392                         .ivsize         = TF_BLOCK_SIZE,
393                         .setkey         = twofish_setkey,
394                         .encrypt        = ctr_crypt,
395                         .decrypt        = ctr_crypt,
396                 },
397         },
398 }, {
399         .cra_name               = "__lrw-twofish-avx",
400         .cra_driver_name        = "__driver-lrw-twofish-avx",
401         .cra_priority           = 0,
402         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
403                                   CRYPTO_ALG_INTERNAL,
404         .cra_blocksize          = TF_BLOCK_SIZE,
405         .cra_ctxsize            = sizeof(struct twofish_lrw_ctx),
406         .cra_alignmask          = 0,
407         .cra_type               = &crypto_blkcipher_type,
408         .cra_module             = THIS_MODULE,
409         .cra_exit               = lrw_twofish_exit_tfm,
410         .cra_u = {
411                 .blkcipher = {
412                         .min_keysize    = TF_MIN_KEY_SIZE +
413                                           TF_BLOCK_SIZE,
414                         .max_keysize    = TF_MAX_KEY_SIZE +
415                                           TF_BLOCK_SIZE,
416                         .ivsize         = TF_BLOCK_SIZE,
417                         .setkey         = lrw_twofish_setkey,
418                         .encrypt        = lrw_encrypt,
419                         .decrypt        = lrw_decrypt,
420                 },
421         },
422 }, {
423         .cra_name               = "__xts-twofish-avx",
424         .cra_driver_name        = "__driver-xts-twofish-avx",
425         .cra_priority           = 0,
426         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
427                                   CRYPTO_ALG_INTERNAL,
428         .cra_blocksize          = TF_BLOCK_SIZE,
429         .cra_ctxsize            = sizeof(struct twofish_xts_ctx),
430         .cra_alignmask          = 0,
431         .cra_type               = &crypto_blkcipher_type,
432         .cra_module             = THIS_MODULE,
433         .cra_u = {
434                 .blkcipher = {
435                         .min_keysize    = TF_MIN_KEY_SIZE * 2,
436                         .max_keysize    = TF_MAX_KEY_SIZE * 2,
437                         .ivsize         = TF_BLOCK_SIZE,
438                         .setkey         = xts_twofish_setkey,
439                         .encrypt        = xts_encrypt,
440                         .decrypt        = xts_decrypt,
441                 },
442         },
443 }, {
444         .cra_name               = "ecb(twofish)",
445         .cra_driver_name        = "ecb-twofish-avx",
446         .cra_priority           = 400,
447         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
448         .cra_blocksize          = TF_BLOCK_SIZE,
449         .cra_ctxsize            = sizeof(struct async_helper_ctx),
450         .cra_alignmask          = 0,
451         .cra_type               = &crypto_ablkcipher_type,
452         .cra_module             = THIS_MODULE,
453         .cra_init               = ablk_init,
454         .cra_exit               = ablk_exit,
455         .cra_u = {
456                 .ablkcipher = {
457                         .min_keysize    = TF_MIN_KEY_SIZE,
458                         .max_keysize    = TF_MAX_KEY_SIZE,
459                         .setkey         = ablk_set_key,
460                         .encrypt        = ablk_encrypt,
461                         .decrypt        = ablk_decrypt,
462                 },
463         },
464 }, {
465         .cra_name               = "cbc(twofish)",
466         .cra_driver_name        = "cbc-twofish-avx",
467         .cra_priority           = 400,
468         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
469         .cra_blocksize          = TF_BLOCK_SIZE,
470         .cra_ctxsize            = sizeof(struct async_helper_ctx),
471         .cra_alignmask          = 0,
472         .cra_type               = &crypto_ablkcipher_type,
473         .cra_module             = THIS_MODULE,
474         .cra_init               = ablk_init,
475         .cra_exit               = ablk_exit,
476         .cra_u = {
477                 .ablkcipher = {
478                         .min_keysize    = TF_MIN_KEY_SIZE,
479                         .max_keysize    = TF_MAX_KEY_SIZE,
480                         .ivsize         = TF_BLOCK_SIZE,
481                         .setkey         = ablk_set_key,
482                         .encrypt        = __ablk_encrypt,
483                         .decrypt        = ablk_decrypt,
484                 },
485         },
486 }, {
487         .cra_name               = "ctr(twofish)",
488         .cra_driver_name        = "ctr-twofish-avx",
489         .cra_priority           = 400,
490         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
491         .cra_blocksize          = 1,
492         .cra_ctxsize            = sizeof(struct async_helper_ctx),
493         .cra_alignmask          = 0,
494         .cra_type               = &crypto_ablkcipher_type,
495         .cra_module             = THIS_MODULE,
496         .cra_init               = ablk_init,
497         .cra_exit               = ablk_exit,
498         .cra_u = {
499                 .ablkcipher = {
500                         .min_keysize    = TF_MIN_KEY_SIZE,
501                         .max_keysize    = TF_MAX_KEY_SIZE,
502                         .ivsize         = TF_BLOCK_SIZE,
503                         .setkey         = ablk_set_key,
504                         .encrypt        = ablk_encrypt,
505                         .decrypt        = ablk_encrypt,
506                         .geniv          = "chainiv",
507                 },
508         },
509 }, {
510         .cra_name               = "lrw(twofish)",
511         .cra_driver_name        = "lrw-twofish-avx",
512         .cra_priority           = 400,
513         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
514         .cra_blocksize          = TF_BLOCK_SIZE,
515         .cra_ctxsize            = sizeof(struct async_helper_ctx),
516         .cra_alignmask          = 0,
517         .cra_type               = &crypto_ablkcipher_type,
518         .cra_module             = THIS_MODULE,
519         .cra_init               = ablk_init,
520         .cra_exit               = ablk_exit,
521         .cra_u = {
522                 .ablkcipher = {
523                         .min_keysize    = TF_MIN_KEY_SIZE +
524                                           TF_BLOCK_SIZE,
525                         .max_keysize    = TF_MAX_KEY_SIZE +
526                                           TF_BLOCK_SIZE,
527                         .ivsize         = TF_BLOCK_SIZE,
528                         .setkey         = ablk_set_key,
529                         .encrypt        = ablk_encrypt,
530                         .decrypt        = ablk_decrypt,
531                 },
532         },
533 }, {
534         .cra_name               = "xts(twofish)",
535         .cra_driver_name        = "xts-twofish-avx",
536         .cra_priority           = 400,
537         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
538         .cra_blocksize          = TF_BLOCK_SIZE,
539         .cra_ctxsize            = sizeof(struct async_helper_ctx),
540         .cra_alignmask          = 0,
541         .cra_type               = &crypto_ablkcipher_type,
542         .cra_module             = THIS_MODULE,
543         .cra_init               = ablk_init,
544         .cra_exit               = ablk_exit,
545         .cra_u = {
546                 .ablkcipher = {
547                         .min_keysize    = TF_MIN_KEY_SIZE * 2,
548                         .max_keysize    = TF_MAX_KEY_SIZE * 2,
549                         .ivsize         = TF_BLOCK_SIZE,
550                         .setkey         = ablk_set_key,
551                         .encrypt        = ablk_encrypt,
552                         .decrypt        = ablk_decrypt,
553                 },
554         },
555 } };
556
557 static int __init twofish_init(void)
558 {
559         const char *feature_name;
560
561         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, &feature_name)) {
562                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
563                 return -ENODEV;
564         }
565
566         return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
567 }
568
569 static void __exit twofish_exit(void)
570 {
571         crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
572 }
573
574 module_init(twofish_init);
575 module_exit(twofish_exit);
576
577 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
578 MODULE_LICENSE("GPL");
579 MODULE_ALIAS_CRYPTO("twofish");