drivers/crypto/inside-secure/safexcel_cipher.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2017 Marvell
   4  *
   5  * Antoine Tenart <antoine.tenart@free-electrons.com>
   6  */
   7
   8 #include <linux/device.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/dmapool.h>
  11
  12 #include <crypto/aead.h>
  13 #include <crypto/aes.h>
  14 #include <crypto/authenc.h>
  15 #include <crypto/des.h>
  16 #include <crypto/sha.h>
  17 #include <crypto/skcipher.h>
  18 #include <crypto/internal/aead.h>
  19 #include <crypto/internal/skcipher.h>
  20
  21 #include "safexcel.h"
  22
  23 enum safexcel_cipher_direction {
  24         SAFEXCEL_ENCRYPT,
  25         SAFEXCEL_DECRYPT,
  26 };
  27
  28 enum safexcel_cipher_alg {
  29         SAFEXCEL_DES,
  30         SAFEXCEL_3DES,
  31         SAFEXCEL_AES,
  32 };
  33
  34 struct safexcel_cipher_ctx {
  35         struct safexcel_context base;
  36         struct safexcel_crypto_priv *priv;
  37
  38         u32 mode;
  39         enum safexcel_cipher_alg alg;
  40         bool aead;
  41
  42         __le32 key[8];
  43         unsigned int key_len;
  44
  45         /* All the below is AEAD specific */
  46         u32 hash_alg;
  47         u32 state_sz;
  48         u32 ipad[SHA512_DIGEST_SIZE / sizeof(u32)];
  49         u32 opad[SHA512_DIGEST_SIZE / sizeof(u32)];
  50 };
  51
  52 struct safexcel_cipher_req {
  53         enum safexcel_cipher_direction direction;
  54         /* Number of result descriptors associated to the request */
  55         unsigned int rdescs;
  56         bool needs_inv;
  57 };
  58
  59 static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
  60                                     struct safexcel_command_desc *cdesc,
  61                                     u32 length)
  62 {
  63         struct safexcel_token *token;
  64         unsigned offset = 0;
  65
  66         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
  67                 switch (ctx->alg) {
  68                 case SAFEXCEL_DES:
  69                         offset = DES_BLOCK_SIZE / sizeof(u32);
  70                         memcpy(cdesc->control_data.token, iv, DES_BLOCK_SIZE);
  71                         cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
  72                         break;
  73                 case SAFEXCEL_3DES:
  74                         offset = DES3_EDE_BLOCK_SIZE / sizeof(u32);
  75                         memcpy(cdesc->control_data.token, iv, DES3_EDE_BLOCK_SIZE);
  76                         cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD;
  77                         break;
  78
  79                 case SAFEXCEL_AES:
  80                         offset = AES_BLOCK_SIZE / sizeof(u32);
  81                         memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
  82                         cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
  83                         break;
  84                 }
  85         }
  86
  87         token = (struct safexcel_token *)(cdesc->control_data.token + offset);
  88
  89         token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
  90         token[0].packet_length = length;
  91         token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET |
  92                         EIP197_TOKEN_STAT_LAST_HASH;
  93         token[0].instructions = EIP197_TOKEN_INS_LAST |
  94                                 EIP197_TOKEN_INS_TYPE_CRYTO |
  95                                 EIP197_TOKEN_INS_TYPE_OUTPUT;
  96 }
  97
  98 static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv,
  99                                 struct safexcel_command_desc *cdesc,
 100                                 enum safexcel_cipher_direction direction,
 101                                 u32 cryptlen, u32 assoclen, u32 digestsize)
 102 {
 103         struct safexcel_token *token;
 104         unsigned offset = 0;
 105
 106         if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) {
 107                 offset = AES_BLOCK_SIZE / sizeof(u32);
 108                 memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE);
 109
 110                 cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD;
 111         }
 112
 113         token = (struct safexcel_token *)(cdesc->control_data.token + offset);
 114
 115         if (direction == SAFEXCEL_DECRYPT)
 116                 cryptlen -= digestsize;
 117
 118         token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 119         token[0].packet_length = assoclen;
 120         token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH |
 121                                 EIP197_TOKEN_INS_TYPE_OUTPUT;
 122
 123         token[1].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
 124         token[1].packet_length = cryptlen;
 125         token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
 126         token[1].instructions = EIP197_TOKEN_INS_LAST |
 127                                 EIP197_TOKEN_INS_TYPE_CRYTO |
 128                                 EIP197_TOKEN_INS_TYPE_HASH |
 129                                 EIP197_TOKEN_INS_TYPE_OUTPUT;
 130
 131         if (direction == SAFEXCEL_ENCRYPT) {
 132                 token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
 133                 token[2].packet_length = digestsize;
 134                 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
 135                                 EIP197_TOKEN_STAT_LAST_PACKET;
 136                 token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
 137                                         EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 138         } else {
 139                 token[2].opcode = EIP197_TOKEN_OPCODE_RETRIEVE;
 140                 token[2].packet_length = digestsize;
 141                 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
 142                                 EIP197_TOKEN_STAT_LAST_PACKET;
 143                 token[2].instructions = EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
 144
 145                 token[3].opcode = EIP197_TOKEN_OPCODE_VERIFY;
 146                 token[3].packet_length = digestsize |
 147                                          EIP197_TOKEN_HASH_RESULT_VERIFY;
 148                 token[3].stat = EIP197_TOKEN_STAT_LAST_HASH |
 149                                 EIP197_TOKEN_STAT_LAST_PACKET;
 150                 token[3].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT;
 151         }
 152 }
 153
 154 static int safexcel_skcipher_aes_setkey(struct crypto_skcipher *ctfm,
 155                                         const u8 *key, unsigned int len)
 156 {
 157         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 158         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 159         struct safexcel_crypto_priv *priv = ctx->priv;
 160         struct crypto_aes_ctx aes;
 161         int ret, i;
 162
 163         ret = crypto_aes_expand_key(&aes, key, len);
 164         if (ret) {
 165                 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 166                 return ret;
 167         }
 168
 169         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma) {
 170                 for (i = 0; i < len / sizeof(u32); i++) {
 171                         if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) {
 172                                 ctx->base.needs_inv = true;
 173                                 break;
 174                         }
 175                 }
 176         }
 177
 178         for (i = 0; i < len / sizeof(u32); i++)
 179                 ctx->key[i] = cpu_to_le32(aes.key_enc[i]);
 180
 181         ctx->key_len = len;
 182
 183         memzero_explicit(&aes, sizeof(aes));
 184         return 0;
 185 }
 186
 187 static int safexcel_aead_aes_setkey(struct crypto_aead *ctfm, const u8 *key,
 188                                     unsigned int len)
 189 {
 190         struct crypto_tfm *tfm = crypto_aead_tfm(ctfm);
 191         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 192         struct safexcel_ahash_export_state istate, ostate;
 193         struct safexcel_crypto_priv *priv = ctx->priv;
 194         struct crypto_authenc_keys keys;
 195
 196         if (crypto_authenc_extractkeys(&keys, key, len) != 0)
 197                 goto badkey;
 198
 199         if (keys.enckeylen > sizeof(ctx->key))
 200                 goto badkey;
 201
 202         /* Encryption key */
 203         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
 204             memcmp(ctx->key, keys.enckey, keys.enckeylen))
 205                 ctx->base.needs_inv = true;
 206
 207         /* Auth key */
 208         switch (ctx->hash_alg) {
 209         case CONTEXT_CONTROL_CRYPTO_ALG_SHA1:
 210                 if (safexcel_hmac_setkey("safexcel-sha1", keys.authkey,
 211                                          keys.authkeylen, &istate, &ostate))
 212                         goto badkey;
 213                 break;
 214         case CONTEXT_CONTROL_CRYPTO_ALG_SHA224:
 215                 if (safexcel_hmac_setkey("safexcel-sha224", keys.authkey,
 216                                          keys.authkeylen, &istate, &ostate))
 217                         goto badkey;
 218                 break;
 219         case CONTEXT_CONTROL_CRYPTO_ALG_SHA256:
 220                 if (safexcel_hmac_setkey("safexcel-sha256", keys.authkey,
 221                                          keys.authkeylen, &istate, &ostate))
 222                         goto badkey;
 223                 break;
 224         case CONTEXT_CONTROL_CRYPTO_ALG_SHA384:
 225                 if (safexcel_hmac_setkey("safexcel-sha384", keys.authkey,
 226                                          keys.authkeylen, &istate, &ostate))
 227                         goto badkey;
 228                 break;
 229         case CONTEXT_CONTROL_CRYPTO_ALG_SHA512:
 230                 if (safexcel_hmac_setkey("safexcel-sha512", keys.authkey,
 231                                          keys.authkeylen, &istate, &ostate))
 232                         goto badkey;
 233                 break;
 234         default:
 235                 dev_err(priv->dev, "aead: unsupported hash algorithm\n");
 236                 goto badkey;
 237         }
 238
 239         crypto_aead_set_flags(ctfm, crypto_aead_get_flags(ctfm) &
 240                                     CRYPTO_TFM_RES_MASK);
 241
 242         if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr_dma &&
 243             (memcmp(ctx->ipad, istate.state, ctx->state_sz) ||
 244              memcmp(ctx->opad, ostate.state, ctx->state_sz)))
 245                 ctx->base.needs_inv = true;
 246
 247         /* Now copy the keys into the context */
 248         memcpy(ctx->key, keys.enckey, keys.enckeylen);
 249         ctx->key_len = keys.enckeylen;
 250
 251         memcpy(ctx->ipad, &istate.state, ctx->state_sz);
 252         memcpy(ctx->opad, &ostate.state, ctx->state_sz);
 253
 254         memzero_explicit(&keys, sizeof(keys));
 255         return 0;
 256
 257 badkey:
 258         crypto_aead_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 259         memzero_explicit(&keys, sizeof(keys));
 260         return -EINVAL;
 261 }
 262
 263 static int safexcel_context_control(struct safexcel_cipher_ctx *ctx,
 264                                     struct crypto_async_request *async,
 265                                     struct safexcel_cipher_req *sreq,
 266                                     struct safexcel_command_desc *cdesc)
 267 {
 268         struct safexcel_crypto_priv *priv = ctx->priv;
 269         int ctrl_size;
 270
 271         if (ctx->aead) {
 272                 if (sreq->direction == SAFEXCEL_ENCRYPT)
 273                         cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_ENCRYPT_HASH_OUT;
 274                 else
 275                         cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_HASH_DECRYPT_IN;
 276         } else {
 277                 cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT;
 278
 279                 /* The decryption control type is a combination of the
 280                  * encryption type and CONTEXT_CONTROL_TYPE_NULL_IN, for all
 281                  * types.
 282                  */
 283                 if (sreq->direction == SAFEXCEL_DECRYPT)
 284                         cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_NULL_IN;
 285         }
 286
 287         cdesc->control_data.control0 |= CONTEXT_CONTROL_KEY_EN;
 288         cdesc->control_data.control1 |= ctx->mode;
 289
 290         if (ctx->aead)
 291                 cdesc->control_data.control0 |= CONTEXT_CONTROL_DIGEST_HMAC |
 292                                                 ctx->hash_alg;
 293
 294         if (ctx->alg == SAFEXCEL_DES) {
 295                 cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_DES;
 296         } else if (ctx->alg == SAFEXCEL_3DES) {
 297                 cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_3DES;
 298         } else if (ctx->alg == SAFEXCEL_AES) {
 299                 switch (ctx->key_len) {
 300                 case AES_KEYSIZE_128:
 301                         cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128;
 302                         break;
 303                 case AES_KEYSIZE_192:
 304                         cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192;
 305                         break;
 306                 case AES_KEYSIZE_256:
 307                         cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256;
 308                         break;
 309                 default:
 310                         dev_err(priv->dev, "aes keysize not supported: %u\n",
 311                                 ctx->key_len);
 312                         return -EINVAL;
 313                 }
 314         }
 315
 316         ctrl_size = ctx->key_len / sizeof(u32);
 317         if (ctx->aead)
 318                 /* Take in account the ipad+opad digests */
 319                 ctrl_size += ctx->state_sz / sizeof(u32) * 2;
 320         cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(ctrl_size);
 321
 322         return 0;
 323 }
 324
 325 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int ring,
 326                                       struct crypto_async_request *async,
 327                                       struct scatterlist *src,
 328                                       struct scatterlist *dst,
 329                                       unsigned int cryptlen,
 330                                       struct safexcel_cipher_req *sreq,
 331                                       bool *should_complete, int *ret)
 332 {
 333         struct safexcel_result_desc *rdesc;
 334         int ndesc = 0;
 335
 336         *ret = 0;
 337
 338         if (unlikely(!sreq->rdescs))
 339                 return 0;
 340
 341         while (sreq->rdescs--) {
 342                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 343                 if (IS_ERR(rdesc)) {
 344                         dev_err(priv->dev,
 345                                 "cipher: result: could not retrieve the result descriptor\n");
 346                         *ret = PTR_ERR(rdesc);
 347                         break;
 348                 }
 349
 350                 if (likely(!*ret))
 351                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 352
 353                 ndesc++;
 354         }
 355
 356         safexcel_complete(priv, ring);
 357
 358         if (src == dst) {
 359                 dma_unmap_sg(priv->dev, src,
 360                              sg_nents_for_len(src, cryptlen),
 361                              DMA_BIDIRECTIONAL);
 362         } else {
 363                 dma_unmap_sg(priv->dev, src,
 364                              sg_nents_for_len(src, cryptlen),
 365                              DMA_TO_DEVICE);
 366                 dma_unmap_sg(priv->dev, dst,
 367                              sg_nents_for_len(dst, cryptlen),
 368                              DMA_FROM_DEVICE);
 369         }
 370
 371         *should_complete = true;
 372
 373         return ndesc;
 374 }
 375
 376 static int safexcel_send_req(struct crypto_async_request *base, int ring,
 377                              struct safexcel_cipher_req *sreq,
 378                              struct scatterlist *src, struct scatterlist *dst,
 379                              unsigned int cryptlen, unsigned int assoclen,
 380                              unsigned int digestsize, u8 *iv, int *commands,
 381                              int *results)
 382 {
 383         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 384         struct safexcel_crypto_priv *priv = ctx->priv;
 385         struct safexcel_command_desc *cdesc;
 386         struct safexcel_result_desc *rdesc, *first_rdesc = NULL;
 387         struct scatterlist *sg;
 388         unsigned int totlen = cryptlen + assoclen;
 389         int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = totlen;
 390         int i, ret = 0;
 391
 392         if (src == dst) {
 393                 nr_src = dma_map_sg(priv->dev, src,
 394                                     sg_nents_for_len(src, totlen),
 395                                     DMA_BIDIRECTIONAL);
 396                 nr_dst = nr_src;
 397                 if (!nr_src)
 398                         return -EINVAL;
 399         } else {
 400                 nr_src = dma_map_sg(priv->dev, src,
 401                                     sg_nents_for_len(src, totlen),
 402                                     DMA_TO_DEVICE);
 403                 if (!nr_src)
 404                         return -EINVAL;
 405
 406                 nr_dst = dma_map_sg(priv->dev, dst,
 407                                     sg_nents_for_len(dst, totlen),
 408                                     DMA_FROM_DEVICE);
 409                 if (!nr_dst) {
 410                         dma_unmap_sg(priv->dev, src,
 411                                      sg_nents_for_len(src, totlen),
 412                                      DMA_TO_DEVICE);
 413                         return -EINVAL;
 414                 }
 415         }
 416
 417         memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len);
 418
 419         if (ctx->aead) {
 420                 memcpy(ctx->base.ctxr->data + ctx->key_len / sizeof(u32),
 421                        ctx->ipad, ctx->state_sz);
 422                 memcpy(ctx->base.ctxr->data + (ctx->key_len + ctx->state_sz) / sizeof(u32),
 423                        ctx->opad, ctx->state_sz);
 424         }
 425
 426         /* command descriptors */
 427         for_each_sg(src, sg, nr_src, i) {
 428                 int len = sg_dma_len(sg);
 429
 430                 /* Do not overflow the request */
 431                 if (queued - len < 0)
 432                         len = queued;
 433
 434                 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - len),
 435                                            sg_dma_address(sg), len, totlen,
 436                                            ctx->base.ctxr_dma);
 437                 if (IS_ERR(cdesc)) {
 438                         /* No space left in the command descriptor ring */
 439                         ret = PTR_ERR(cdesc);
 440                         goto cdesc_rollback;
 441                 }
 442                 n_cdesc++;
 443
 444                 if (n_cdesc == 1) {
 445                         safexcel_context_control(ctx, base, sreq, cdesc);
 446                         if (ctx->aead)
 447                                 safexcel_aead_token(ctx, iv, cdesc,
 448                                                     sreq->direction, cryptlen,
 449                                                     assoclen, digestsize);
 450                         else
 451                                 safexcel_skcipher_token(ctx, iv, cdesc,
 452                                                         cryptlen);
 453                 }
 454
 455                 queued -= len;
 456                 if (!queued)
 457                         break;
 458         }
 459
 460         /* result descriptors */
 461         for_each_sg(dst, sg, nr_dst, i) {
 462                 bool first = !i, last = (i == nr_dst - 1);
 463                 u32 len = sg_dma_len(sg);
 464
 465                 rdesc = safexcel_add_rdesc(priv, ring, first, last,
 466                                            sg_dma_address(sg), len);
 467                 if (IS_ERR(rdesc)) {
 468                         /* No space left in the result descriptor ring */
 469                         ret = PTR_ERR(rdesc);
 470                         goto rdesc_rollback;
 471                 }
 472                 if (first)
 473                         first_rdesc = rdesc;
 474                 n_rdesc++;
 475         }
 476
 477         safexcel_rdr_req_set(priv, ring, first_rdesc, base);
 478
 479         *commands = n_cdesc;
 480         *results = n_rdesc;
 481         return 0;
 482
 483 rdesc_rollback:
 484         for (i = 0; i < n_rdesc; i++)
 485                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr);
 486 cdesc_rollback:
 487         for (i = 0; i < n_cdesc; i++)
 488                 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
 489
 490         if (src == dst) {
 491                 dma_unmap_sg(priv->dev, src,
 492                              sg_nents_for_len(src, totlen),
 493                              DMA_BIDIRECTIONAL);
 494         } else {
 495                 dma_unmap_sg(priv->dev, src,
 496                              sg_nents_for_len(src, totlen),
 497                              DMA_TO_DEVICE);
 498                 dma_unmap_sg(priv->dev, dst,
 499                              sg_nents_for_len(dst, totlen),
 500                              DMA_FROM_DEVICE);
 501         }
 502
 503         return ret;
 504 }
 505
 506 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
 507                                       int ring,
 508                                       struct crypto_async_request *base,
 509                                       struct safexcel_cipher_req *sreq,
 510                                       bool *should_complete, int *ret)
 511 {
 512         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 513         struct safexcel_result_desc *rdesc;
 514         int ndesc = 0, enq_ret;
 515
 516         *ret = 0;
 517
 518         if (unlikely(!sreq->rdescs))
 519                 return 0;
 520
 521         while (sreq->rdescs--) {
 522                 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
 523                 if (IS_ERR(rdesc)) {
 524                         dev_err(priv->dev,
 525                                 "cipher: invalidate: could not retrieve the result descriptor\n");
 526                         *ret = PTR_ERR(rdesc);
 527                         break;
 528                 }
 529
 530                 if (likely(!*ret))
 531                         *ret = safexcel_rdesc_check_errors(priv, rdesc);
 532
 533                 ndesc++;
 534         }
 535
 536         safexcel_complete(priv, ring);
 537
 538         if (ctx->base.exit_inv) {
 539                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 540                               ctx->base.ctxr_dma);
 541
 542                 *should_complete = true;
 543
 544                 return ndesc;
 545         }
 546
 547         ring = safexcel_select_ring(priv);
 548         ctx->base.ring = ring;
 549
 550         spin_lock_bh(&priv->ring[ring].queue_lock);
 551         enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
 552         spin_unlock_bh(&priv->ring[ring].queue_lock);
 553
 554         if (enq_ret != -EINPROGRESS)
 555                 *ret = enq_ret;
 556
 557         queue_work(priv->ring[ring].workqueue,
 558                    &priv->ring[ring].work_data.work);
 559
 560         *should_complete = false;
 561
 562         return ndesc;
 563 }
 564
 565 static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv,
 566                                            int ring,
 567                                            struct crypto_async_request *async,
 568                                            bool *should_complete, int *ret)
 569 {
 570         struct skcipher_request *req = skcipher_request_cast(async);
 571         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 572         int err;
 573
 574         if (sreq->needs_inv) {
 575                 sreq->needs_inv = false;
 576                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
 577                                                  should_complete, ret);
 578         } else {
 579                 err = safexcel_handle_req_result(priv, ring, async, req->src,
 580                                                  req->dst, req->cryptlen, sreq,
 581                                                  should_complete, ret);
 582         }
 583
 584         return err;
 585 }
 586
 587 static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv,
 588                                        int ring,
 589                                        struct crypto_async_request *async,
 590                                        bool *should_complete, int *ret)
 591 {
 592         struct aead_request *req = aead_request_cast(async);
 593         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 594         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
 595         int err;
 596
 597         if (sreq->needs_inv) {
 598                 sreq->needs_inv = false;
 599                 err = safexcel_handle_inv_result(priv, ring, async, sreq,
 600                                                  should_complete, ret);
 601         } else {
 602                 err = safexcel_handle_req_result(priv, ring, async, req->src,
 603                                                  req->dst,
 604                                                  req->cryptlen + crypto_aead_authsize(tfm),
 605                                                  sreq, should_complete, ret);
 606         }
 607
 608         return err;
 609 }
 610
 611 static int safexcel_cipher_send_inv(struct crypto_async_request *base,
 612                                     int ring, int *commands, int *results)
 613 {
 614         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 615         struct safexcel_crypto_priv *priv = ctx->priv;
 616         int ret;
 617
 618         ret = safexcel_invalidate_cache(base, priv, ctx->base.ctxr_dma, ring);
 619         if (unlikely(ret))
 620                 return ret;
 621
 622         *commands = 1;
 623         *results = 1;
 624
 625         return 0;
 626 }
 627
 628 static int safexcel_skcipher_send(struct crypto_async_request *async, int ring,
 629                                   int *commands, int *results)
 630 {
 631         struct skcipher_request *req = skcipher_request_cast(async);
 632         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 633         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 634         struct safexcel_crypto_priv *priv = ctx->priv;
 635         int ret;
 636
 637         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
 638
 639         if (sreq->needs_inv)
 640                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
 641         else
 642                 ret = safexcel_send_req(async, ring, sreq, req->src,
 643                                         req->dst, req->cryptlen, 0, 0, req->iv,
 644                                         commands, results);
 645
 646         sreq->rdescs = *results;
 647         return ret;
 648 }
 649
 650 static int safexcel_aead_send(struct crypto_async_request *async, int ring,
 651                               int *commands, int *results)
 652 {
 653         struct aead_request *req = aead_request_cast(async);
 654         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 655         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 656         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
 657         struct safexcel_crypto_priv *priv = ctx->priv;
 658         int ret;
 659
 660         BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && sreq->needs_inv);
 661
 662         if (sreq->needs_inv)
 663                 ret = safexcel_cipher_send_inv(async, ring, commands, results);
 664         else
 665                 ret = safexcel_send_req(async, ring, sreq, req->src, req->dst,
 666                                         req->cryptlen, req->assoclen,
 667                                         crypto_aead_authsize(tfm), req->iv,
 668                                         commands, results);
 669         sreq->rdescs = *results;
 670         return ret;
 671 }
 672
 673 static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm,
 674                                     struct crypto_async_request *base,
 675                                     struct safexcel_cipher_req *sreq,
 676                                     struct safexcel_inv_result *result)
 677 {
 678         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 679         struct safexcel_crypto_priv *priv = ctx->priv;
 680         int ring = ctx->base.ring;
 681
 682         init_completion(&result->completion);
 683
 684         ctx = crypto_tfm_ctx(base->tfm);
 685         ctx->base.exit_inv = true;
 686         sreq->needs_inv = true;
 687
 688         spin_lock_bh(&priv->ring[ring].queue_lock);
 689         crypto_enqueue_request(&priv->ring[ring].queue, base);
 690         spin_unlock_bh(&priv->ring[ring].queue_lock);
 691
 692         queue_work(priv->ring[ring].workqueue,
 693                    &priv->ring[ring].work_data.work);
 694
 695         wait_for_completion(&result->completion);
 696
 697         if (result->error) {
 698                 dev_warn(priv->dev,
 699                         "cipher: sync: invalidate: completion error %d\n",
 700                          result->error);
 701                 return result->error;
 702         }
 703
 704         return 0;
 705 }
 706
 707 static int safexcel_skcipher_exit_inv(struct crypto_tfm *tfm)
 708 {
 709         EIP197_REQUEST_ON_STACK(req, skcipher, EIP197_SKCIPHER_REQ_SIZE);
 710         struct safexcel_cipher_req *sreq = skcipher_request_ctx(req);
 711         struct safexcel_inv_result result = {};
 712
 713         memset(req, 0, sizeof(struct skcipher_request));
 714
 715         skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 716                                       safexcel_inv_complete, &result);
 717         skcipher_request_set_tfm(req, __crypto_skcipher_cast(tfm));
 718
 719         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
 720 }
 721
 722 static int safexcel_aead_exit_inv(struct crypto_tfm *tfm)
 723 {
 724         EIP197_REQUEST_ON_STACK(req, aead, EIP197_AEAD_REQ_SIZE);
 725         struct safexcel_cipher_req *sreq = aead_request_ctx(req);
 726         struct safexcel_inv_result result = {};
 727
 728         memset(req, 0, sizeof(struct aead_request));
 729
 730         aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 731                                   safexcel_inv_complete, &result);
 732         aead_request_set_tfm(req, __crypto_aead_cast(tfm));
 733
 734         return safexcel_cipher_exit_inv(tfm, &req->base, sreq, &result);
 735 }
 736
 737 static int safexcel_queue_req(struct crypto_async_request *base,
 738                         struct safexcel_cipher_req *sreq,
 739                         enum safexcel_cipher_direction dir, u32 mode,
 740                         enum safexcel_cipher_alg alg)
 741 {
 742         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
 743         struct safexcel_crypto_priv *priv = ctx->priv;
 744         int ret, ring;
 745
 746         sreq->needs_inv = false;
 747         sreq->direction = dir;
 748         ctx->alg = alg;
 749         ctx->mode = mode;
 750
 751         if (ctx->base.ctxr) {
 752                 if (priv->flags & EIP197_TRC_CACHE && ctx->base.needs_inv) {
 753                         sreq->needs_inv = true;
 754                         ctx->base.needs_inv = false;
 755                 }
 756         } else {
 757                 ctx->base.ring = safexcel_select_ring(priv);
 758                 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
 759                                                  EIP197_GFP_FLAGS(*base),
 760                                                  &ctx->base.ctxr_dma);
 761                 if (!ctx->base.ctxr)
 762                         return -ENOMEM;
 763         }
 764
 765         ring = ctx->base.ring;
 766
 767         spin_lock_bh(&priv->ring[ring].queue_lock);
 768         ret = crypto_enqueue_request(&priv->ring[ring].queue, base);
 769         spin_unlock_bh(&priv->ring[ring].queue_lock);
 770
 771         queue_work(priv->ring[ring].workqueue,
 772                    &priv->ring[ring].work_data.work);
 773
 774         return ret;
 775 }
 776
 777 static int safexcel_ecb_aes_encrypt(struct skcipher_request *req)
 778 {
 779         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
 780                         SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
 781                         SAFEXCEL_AES);
 782 }
 783
 784 static int safexcel_ecb_aes_decrypt(struct skcipher_request *req)
 785 {
 786         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
 787                         SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
 788                         SAFEXCEL_AES);
 789 }
 790
 791 static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm)
 792 {
 793         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 794         struct safexcel_alg_template *tmpl =
 795                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
 796                              alg.skcipher.base);
 797
 798         crypto_skcipher_set_reqsize(__crypto_skcipher_cast(tfm),
 799                                     sizeof(struct safexcel_cipher_req));
 800
 801         ctx->priv = tmpl->priv;
 802
 803         ctx->base.send = safexcel_skcipher_send;
 804         ctx->base.handle_result = safexcel_skcipher_handle_result;
 805         return 0;
 806 }
 807
 808 static int safexcel_cipher_cra_exit(struct crypto_tfm *tfm)
 809 {
 810         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 811
 812         memzero_explicit(ctx->key, sizeof(ctx->key));
 813
 814         /* context not allocated, skip invalidation */
 815         if (!ctx->base.ctxr)
 816                 return -ENOMEM;
 817
 818         memzero_explicit(ctx->base.ctxr->data, sizeof(ctx->base.ctxr->data));
 819         return 0;
 820 }
 821
 822 static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm)
 823 {
 824         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 825         struct safexcel_crypto_priv *priv = ctx->priv;
 826         int ret;
 827
 828         if (safexcel_cipher_cra_exit(tfm))
 829                 return;
 830
 831         if (priv->flags & EIP197_TRC_CACHE) {
 832                 ret = safexcel_skcipher_exit_inv(tfm);
 833                 if (ret)
 834                         dev_warn(priv->dev, "skcipher: invalidation error %d\n",
 835                                  ret);
 836         } else {
 837                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 838                               ctx->base.ctxr_dma);
 839         }
 840 }
 841
 842 static void safexcel_aead_cra_exit(struct crypto_tfm *tfm)
 843 {
 844         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 845         struct safexcel_crypto_priv *priv = ctx->priv;
 846         int ret;
 847
 848         if (safexcel_cipher_cra_exit(tfm))
 849                 return;
 850
 851         if (priv->flags & EIP197_TRC_CACHE) {
 852                 ret = safexcel_aead_exit_inv(tfm);
 853                 if (ret)
 854                         dev_warn(priv->dev, "aead: invalidation error %d\n",
 855                                  ret);
 856         } else {
 857                 dma_pool_free(priv->context_pool, ctx->base.ctxr,
 858                               ctx->base.ctxr_dma);
 859         }
 860 }
 861
 862 struct safexcel_alg_template safexcel_alg_ecb_aes = {
 863         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
 864         .engines = EIP97IES | EIP197B | EIP197D,
 865         .alg.skcipher = {
 866                 .setkey = safexcel_skcipher_aes_setkey,
 867                 .encrypt = safexcel_ecb_aes_encrypt,
 868                 .decrypt = safexcel_ecb_aes_decrypt,
 869                 .min_keysize = AES_MIN_KEY_SIZE,
 870                 .max_keysize = AES_MAX_KEY_SIZE,
 871                 .base = {
 872                         .cra_name = "ecb(aes)",
 873                         .cra_driver_name = "safexcel-ecb-aes",
 874                         .cra_priority = 300,
 875                         .cra_flags = CRYPTO_ALG_ASYNC |
 876                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
 877                         .cra_blocksize = AES_BLOCK_SIZE,
 878                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
 879                         .cra_alignmask = 0,
 880                         .cra_init = safexcel_skcipher_cra_init,
 881                         .cra_exit = safexcel_skcipher_cra_exit,
 882                         .cra_module = THIS_MODULE,
 883                 },
 884         },
 885 };
 886
 887 static int safexcel_cbc_aes_encrypt(struct skcipher_request *req)
 888 {
 889         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
 890                         SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
 891                         SAFEXCEL_AES);
 892 }
 893
 894 static int safexcel_cbc_aes_decrypt(struct skcipher_request *req)
 895 {
 896         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
 897                         SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
 898                         SAFEXCEL_AES);
 899 }
 900
 901 struct safexcel_alg_template safexcel_alg_cbc_aes = {
 902         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
 903         .engines = EIP97IES | EIP197B | EIP197D,
 904         .alg.skcipher = {
 905                 .setkey = safexcel_skcipher_aes_setkey,
 906                 .encrypt = safexcel_cbc_aes_encrypt,
 907                 .decrypt = safexcel_cbc_aes_decrypt,
 908                 .min_keysize = AES_MIN_KEY_SIZE,
 909                 .max_keysize = AES_MAX_KEY_SIZE,
 910                 .ivsize = AES_BLOCK_SIZE,
 911                 .base = {
 912                         .cra_name = "cbc(aes)",
 913                         .cra_driver_name = "safexcel-cbc-aes",
 914                         .cra_priority = 300,
 915                         .cra_flags = CRYPTO_ALG_ASYNC |
 916                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
 917                         .cra_blocksize = AES_BLOCK_SIZE,
 918                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
 919                         .cra_alignmask = 0,
 920                         .cra_init = safexcel_skcipher_cra_init,
 921                         .cra_exit = safexcel_skcipher_cra_exit,
 922                         .cra_module = THIS_MODULE,
 923                 },
 924         },
 925 };
 926
 927 static int safexcel_cbc_des_encrypt(struct skcipher_request *req)
 928 {
 929         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
 930                         SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
 931                         SAFEXCEL_DES);
 932 }
 933
 934 static int safexcel_cbc_des_decrypt(struct skcipher_request *req)
 935 {
 936         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
 937                         SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
 938                         SAFEXCEL_DES);
 939 }
 940
 941 static int safexcel_des_setkey(struct crypto_skcipher *ctfm, const u8 *key,
 942                                unsigned int len)
 943 {
 944         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
 945         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
 946         u32 tmp[DES_EXPKEY_WORDS];
 947         int ret;
 948
 949         if (len != DES_KEY_SIZE) {
 950                 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 951                 return -EINVAL;
 952         }
 953
 954         ret = des_ekey(tmp, key);
 955         if (!ret && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
 956                 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
 957                 return -EINVAL;
 958         }
 959
 960         /* if context exits and key changed, need to invalidate it */
 961         if (ctx->base.ctxr_dma)
 962                 if (memcmp(ctx->key, key, len))
 963                         ctx->base.needs_inv = true;
 964
 965         memcpy(ctx->key, key, len);
 966         ctx->key_len = len;
 967
 968         return 0;
 969 }
 970
 971 struct safexcel_alg_template safexcel_alg_cbc_des = {
 972         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
 973         .engines = EIP97IES | EIP197B | EIP197D,
 974         .alg.skcipher = {
 975                 .setkey = safexcel_des_setkey,
 976                 .encrypt = safexcel_cbc_des_encrypt,
 977                 .decrypt = safexcel_cbc_des_decrypt,
 978                 .min_keysize = DES_KEY_SIZE,
 979                 .max_keysize = DES_KEY_SIZE,
 980                 .ivsize = DES_BLOCK_SIZE,
 981                 .base = {
 982                         .cra_name = "cbc(des)",
 983                         .cra_driver_name = "safexcel-cbc-des",
 984                         .cra_priority = 300,
 985                         .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
 986                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
 987                         .cra_blocksize = DES_BLOCK_SIZE,
 988                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
 989                         .cra_alignmask = 0,
 990                         .cra_init = safexcel_skcipher_cra_init,
 991                         .cra_exit = safexcel_skcipher_cra_exit,
 992                         .cra_module = THIS_MODULE,
 993                 },
 994         },
 995 };
 996
 997 static int safexcel_ecb_des_encrypt(struct skcipher_request *req)
 998 {
 999         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1000                         SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
1001                         SAFEXCEL_DES);
1002 }
1003
1004 static int safexcel_ecb_des_decrypt(struct skcipher_request *req)
1005 {
1006         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1007                         SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
1008                         SAFEXCEL_DES);
1009 }
1010
1011 struct safexcel_alg_template safexcel_alg_ecb_des = {
1012         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1013         .engines = EIP97IES | EIP197B | EIP197D,
1014         .alg.skcipher = {
1015                 .setkey = safexcel_des_setkey,
1016                 .encrypt = safexcel_ecb_des_encrypt,
1017                 .decrypt = safexcel_ecb_des_decrypt,
1018                 .min_keysize = DES_KEY_SIZE,
1019                 .max_keysize = DES_KEY_SIZE,
1020                 .ivsize = DES_BLOCK_SIZE,
1021                 .base = {
1022                         .cra_name = "ecb(des)",
1023                         .cra_driver_name = "safexcel-ecb-des",
1024                         .cra_priority = 300,
1025                         .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
1026                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1027                         .cra_blocksize = DES_BLOCK_SIZE,
1028                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1029                         .cra_alignmask = 0,
1030                         .cra_init = safexcel_skcipher_cra_init,
1031                         .cra_exit = safexcel_skcipher_cra_exit,
1032                         .cra_module = THIS_MODULE,
1033                 },
1034         },
1035 };
1036
1037 static int safexcel_cbc_des3_ede_encrypt(struct skcipher_request *req)
1038 {
1039         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1040                         SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
1041                         SAFEXCEL_3DES);
1042 }
1043
1044 static int safexcel_cbc_des3_ede_decrypt(struct skcipher_request *req)
1045 {
1046         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1047                         SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_CBC,
1048                         SAFEXCEL_3DES);
1049 }
1050
1051 static int safexcel_des3_ede_setkey(struct crypto_skcipher *ctfm,
1052                                    const u8 *key, unsigned int len)
1053 {
1054         struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm);
1055         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1056
1057         if (len != DES3_EDE_KEY_SIZE) {
1058                 crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1059                 return -EINVAL;
1060         }
1061
1062         /* if context exits and key changed, need to invalidate it */
1063         if (ctx->base.ctxr_dma) {
1064                 if (memcmp(ctx->key, key, len))
1065                         ctx->base.needs_inv = true;
1066         }
1067
1068         memcpy(ctx->key, key, len);
1069
1070         ctx->key_len = len;
1071
1072         return 0;
1073 }
1074
1075 struct safexcel_alg_template safexcel_alg_cbc_des3_ede = {
1076         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1077         .engines = EIP97IES | EIP197B | EIP197D,
1078         .alg.skcipher = {
1079                 .setkey = safexcel_des3_ede_setkey,
1080                 .encrypt = safexcel_cbc_des3_ede_encrypt,
1081                 .decrypt = safexcel_cbc_des3_ede_decrypt,
1082                 .min_keysize = DES3_EDE_KEY_SIZE,
1083                 .max_keysize = DES3_EDE_KEY_SIZE,
1084                 .ivsize = DES3_EDE_BLOCK_SIZE,
1085                 .base = {
1086                         .cra_name = "cbc(des3_ede)",
1087                         .cra_driver_name = "safexcel-cbc-des3_ede",
1088                         .cra_priority = 300,
1089                         .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
1090                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1091                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1092                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1093                         .cra_alignmask = 0,
1094                         .cra_init = safexcel_skcipher_cra_init,
1095                         .cra_exit = safexcel_skcipher_cra_exit,
1096                         .cra_module = THIS_MODULE,
1097                 },
1098         },
1099 };
1100
1101 static int safexcel_ecb_des3_ede_encrypt(struct skcipher_request *req)
1102 {
1103         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1104                         SAFEXCEL_ENCRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
1105                         SAFEXCEL_3DES);
1106 }
1107
1108 static int safexcel_ecb_des3_ede_decrypt(struct skcipher_request *req)
1109 {
1110         return safexcel_queue_req(&req->base, skcipher_request_ctx(req),
1111                         SAFEXCEL_DECRYPT, CONTEXT_CONTROL_CRYPTO_MODE_ECB,
1112                         SAFEXCEL_3DES);
1113 }
1114
1115 struct safexcel_alg_template safexcel_alg_ecb_des3_ede = {
1116         .type = SAFEXCEL_ALG_TYPE_SKCIPHER,
1117         .engines = EIP97IES | EIP197B | EIP197D,
1118         .alg.skcipher = {
1119                 .setkey = safexcel_des3_ede_setkey,
1120                 .encrypt = safexcel_ecb_des3_ede_encrypt,
1121                 .decrypt = safexcel_ecb_des3_ede_decrypt,
1122                 .min_keysize = DES3_EDE_KEY_SIZE,
1123                 .max_keysize = DES3_EDE_KEY_SIZE,
1124                 .ivsize = DES3_EDE_BLOCK_SIZE,
1125                 .base = {
1126                         .cra_name = "ecb(des3_ede)",
1127                         .cra_driver_name = "safexcel-ecb-des3_ede",
1128                         .cra_priority = 300,
1129                         .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
1130                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1131                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1132                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1133                         .cra_alignmask = 0,
1134                         .cra_init = safexcel_skcipher_cra_init,
1135                         .cra_exit = safexcel_skcipher_cra_exit,
1136                         .cra_module = THIS_MODULE,
1137                 },
1138         },
1139 };
1140
1141 static int safexcel_aead_encrypt(struct aead_request *req)
1142 {
1143         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1144
1145         return safexcel_queue_req(&req->base, creq, SAFEXCEL_ENCRYPT,
1146                         CONTEXT_CONTROL_CRYPTO_MODE_CBC, SAFEXCEL_AES);
1147 }
1148
1149 static int safexcel_aead_decrypt(struct aead_request *req)
1150 {
1151         struct safexcel_cipher_req *creq = aead_request_ctx(req);
1152
1153         return safexcel_queue_req(&req->base, creq, SAFEXCEL_DECRYPT,
1154                         CONTEXT_CONTROL_CRYPTO_MODE_CBC, SAFEXCEL_AES);
1155 }
1156
1157 static int safexcel_aead_cra_init(struct crypto_tfm *tfm)
1158 {
1159         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1160         struct safexcel_alg_template *tmpl =
1161                 container_of(tfm->__crt_alg, struct safexcel_alg_template,
1162                              alg.aead.base);
1163
1164         crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
1165                                 sizeof(struct safexcel_cipher_req));
1166
1167         ctx->priv = tmpl->priv;
1168
1169         ctx->aead = true;
1170         ctx->base.send = safexcel_aead_send;
1171         ctx->base.handle_result = safexcel_aead_handle_result;
1172         return 0;
1173 }
1174
1175 static int safexcel_aead_sha1_cra_init(struct crypto_tfm *tfm)
1176 {
1177         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1178
1179         safexcel_aead_cra_init(tfm);
1180         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1181         ctx->state_sz = SHA1_DIGEST_SIZE;
1182         return 0;
1183 }
1184
1185 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha1_cbc_aes = {
1186         .type = SAFEXCEL_ALG_TYPE_AEAD,
1187         .engines = EIP97IES | EIP197B | EIP197D,
1188         .alg.aead = {
1189                 .setkey = safexcel_aead_aes_setkey,
1190                 .encrypt = safexcel_aead_encrypt,
1191                 .decrypt = safexcel_aead_decrypt,
1192                 .ivsize = AES_BLOCK_SIZE,
1193                 .maxauthsize = SHA1_DIGEST_SIZE,
1194                 .base = {
1195                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
1196                         .cra_driver_name = "safexcel-authenc-hmac-sha1-cbc-aes",
1197                         .cra_priority = 300,
1198                         .cra_flags = CRYPTO_ALG_ASYNC |
1199                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1200                         .cra_blocksize = AES_BLOCK_SIZE,
1201                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1202                         .cra_alignmask = 0,
1203                         .cra_init = safexcel_aead_sha1_cra_init,
1204                         .cra_exit = safexcel_aead_cra_exit,
1205                         .cra_module = THIS_MODULE,
1206                 },
1207         },
1208 };
1209
1210 static int safexcel_aead_sha256_cra_init(struct crypto_tfm *tfm)
1211 {
1212         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1213
1214         safexcel_aead_cra_init(tfm);
1215         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1216         ctx->state_sz = SHA256_DIGEST_SIZE;
1217         return 0;
1218 }
1219
1220 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha256_cbc_aes = {
1221         .type = SAFEXCEL_ALG_TYPE_AEAD,
1222         .engines = EIP97IES | EIP197B | EIP197D,
1223         .alg.aead = {
1224                 .setkey = safexcel_aead_aes_setkey,
1225                 .encrypt = safexcel_aead_encrypt,
1226                 .decrypt = safexcel_aead_decrypt,
1227                 .ivsize = AES_BLOCK_SIZE,
1228                 .maxauthsize = SHA256_DIGEST_SIZE,
1229                 .base = {
1230                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
1231                         .cra_driver_name = "safexcel-authenc-hmac-sha256-cbc-aes",
1232                         .cra_priority = 300,
1233                         .cra_flags = CRYPTO_ALG_ASYNC |
1234                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1235                         .cra_blocksize = AES_BLOCK_SIZE,
1236                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1237                         .cra_alignmask = 0,
1238                         .cra_init = safexcel_aead_sha256_cra_init,
1239                         .cra_exit = safexcel_aead_cra_exit,
1240                         .cra_module = THIS_MODULE,
1241                 },
1242         },
1243 };
1244
1245 static int safexcel_aead_sha224_cra_init(struct crypto_tfm *tfm)
1246 {
1247         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1248
1249         safexcel_aead_cra_init(tfm);
1250         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1251         ctx->state_sz = SHA256_DIGEST_SIZE;
1252         return 0;
1253 }
1254
1255 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha224_cbc_aes = {
1256         .type = SAFEXCEL_ALG_TYPE_AEAD,
1257         .engines = EIP97IES | EIP197B | EIP197D,
1258         .alg.aead = {
1259                 .setkey = safexcel_aead_aes_setkey,
1260                 .encrypt = safexcel_aead_encrypt,
1261                 .decrypt = safexcel_aead_decrypt,
1262                 .ivsize = AES_BLOCK_SIZE,
1263                 .maxauthsize = SHA224_DIGEST_SIZE,
1264                 .base = {
1265                         .cra_name = "authenc(hmac(sha224),cbc(aes))",
1266                         .cra_driver_name = "safexcel-authenc-hmac-sha224-cbc-aes",
1267                         .cra_priority = 300,
1268                         .cra_flags = CRYPTO_ALG_ASYNC |
1269                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1270                         .cra_blocksize = AES_BLOCK_SIZE,
1271                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1272                         .cra_alignmask = 0,
1273                         .cra_init = safexcel_aead_sha224_cra_init,
1274                         .cra_exit = safexcel_aead_cra_exit,
1275                         .cra_module = THIS_MODULE,
1276                 },
1277         },
1278 };
1279
1280 static int safexcel_aead_sha512_cra_init(struct crypto_tfm *tfm)
1281 {
1282         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1283
1284         safexcel_aead_cra_init(tfm);
1285         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1286         ctx->state_sz = SHA512_DIGEST_SIZE;
1287         return 0;
1288 }
1289
1290 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha512_cbc_aes = {
1291         .type = SAFEXCEL_ALG_TYPE_AEAD,
1292         .engines = EIP97IES | EIP197B | EIP197D,
1293         .alg.aead = {
1294                 .setkey = safexcel_aead_aes_setkey,
1295                 .encrypt = safexcel_aead_encrypt,
1296                 .decrypt = safexcel_aead_decrypt,
1297                 .ivsize = AES_BLOCK_SIZE,
1298                 .maxauthsize = SHA512_DIGEST_SIZE,
1299                 .base = {
1300                         .cra_name = "authenc(hmac(sha512),cbc(aes))",
1301                         .cra_driver_name = "safexcel-authenc-hmac-sha512-cbc-aes",
1302                         .cra_priority = 300,
1303                         .cra_flags = CRYPTO_ALG_ASYNC |
1304                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1305                         .cra_blocksize = AES_BLOCK_SIZE,
1306                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1307                         .cra_alignmask = 0,
1308                         .cra_init = safexcel_aead_sha512_cra_init,
1309                         .cra_exit = safexcel_aead_cra_exit,
1310                         .cra_module = THIS_MODULE,
1311                 },
1312         },
1313 };
1314
1315 static int safexcel_aead_sha384_cra_init(struct crypto_tfm *tfm)
1316 {
1317         struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
1318
1319         safexcel_aead_cra_init(tfm);
1320         ctx->hash_alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1321         ctx->state_sz = SHA512_DIGEST_SIZE;
1322         return 0;
1323 }
1324
1325 struct safexcel_alg_template safexcel_alg_authenc_hmac_sha384_cbc_aes = {
1326         .type = SAFEXCEL_ALG_TYPE_AEAD,
1327         .engines = EIP97IES | EIP197B | EIP197D,
1328         .alg.aead = {
1329                 .setkey = safexcel_aead_aes_setkey,
1330                 .encrypt = safexcel_aead_encrypt,
1331                 .decrypt = safexcel_aead_decrypt,
1332                 .ivsize = AES_BLOCK_SIZE,
1333                 .maxauthsize = SHA384_DIGEST_SIZE,
1334                 .base = {
1335                         .cra_name = "authenc(hmac(sha384),cbc(aes))",
1336                         .cra_driver_name = "safexcel-authenc-hmac-sha384-cbc-aes",
1337                         .cra_priority = 300,
1338                         .cra_flags = CRYPTO_ALG_ASYNC |
1339                                      CRYPTO_ALG_KERN_DRIVER_ONLY,
1340                         .cra_blocksize = AES_BLOCK_SIZE,
1341                         .cra_ctxsize = sizeof(struct safexcel_cipher_ctx),
1342                         .cra_alignmask = 0,
1343                         .cra_init = safexcel_aead_sha384_cra_init,
1344                         .cra_exit = safexcel_aead_cra_exit,
1345                         .cra_module = THIS_MODULE,
1346                 },
1347         },
1348 };