drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  * Author: Gary R Hook <gary.hook@amd.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/module.h>
  15 #include <linux/kernel.h>
  16 #include <linux/pci.h>
  17 #include <linux/interrupt.h>
  18 #include <crypto/scatterwalk.h>
  19 #include <crypto/des.h>
  20 #include <linux/ccp.h>
  21
  22 #include "ccp-dev.h"
  23
  24 /* SHA initial context values */
  25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  26         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  27         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  28         cpu_to_be32(SHA1_H4),
  29 };
  30
  31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  32         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  33         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  34         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  35         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  36 };
  37
  38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  39         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  40         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  41         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  42         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  43 };
  44
  45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  46         cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
  47         cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
  48         cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
  49         cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
  50 };
  51
  52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
  53         cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
  54         cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
  55         cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
  56         cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
  57 };
  58
  59 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  60                                         ccp_gen_jobid(ccp) : 0)
  61
  62 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  63 {
  64         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  65 }
  66
  67 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  68 {
  69         if (wa->dma_count)
  70                 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
  71
  72         wa->dma_count = 0;
  73 }
  74
  75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  76                                 struct scatterlist *sg, u64 len,
  77                                 enum dma_data_direction dma_dir)
  78 {
  79         memset(wa, 0, sizeof(*wa));
  80
  81         wa->sg = sg;
  82         if (!sg)
  83                 return 0;
  84
  85         wa->nents = sg_nents_for_len(sg, len);
  86         if (wa->nents < 0)
  87                 return wa->nents;
  88
  89         wa->bytes_left = len;
  90         wa->sg_used = 0;
  91
  92         if (len == 0)
  93                 return 0;
  94
  95         if (dma_dir == DMA_NONE)
  96                 return 0;
  97
  98         wa->dma_sg = sg;
  99         wa->dma_sg_head = sg;
 100         wa->dma_dev = dev;
 101         wa->dma_dir = dma_dir;
 102         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
 103         if (!wa->dma_count)
 104                 return -ENOMEM;
 105
 106         return 0;
 107 }
 108
 109 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
 110 {
 111         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
 112         unsigned int sg_combined_len = 0;
 113
 114         if (!wa->sg)
 115                 return;
 116
 117         wa->sg_used += nbytes;
 118         wa->bytes_left -= nbytes;
 119         if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
 120                 /* Advance to the next DMA scatterlist entry */
 121                 wa->dma_sg = sg_next(wa->dma_sg);
 122
 123                 /* In the case that the DMA mapped scatterlist has entries
 124                  * that have been merged, the non-DMA mapped scatterlist
 125                  * must be advanced multiple times for each merged entry.
 126                  * This ensures that the current non-DMA mapped entry
 127                  * corresponds to the current DMA mapped entry.
 128                  */
 129                 do {
 130                         sg_combined_len += wa->sg->length;
 131                         wa->sg = sg_next(wa->sg);
 132                 } while (wa->sg_used > sg_combined_len);
 133
 134                 wa->sg_used = 0;
 135         }
 136 }
 137
 138 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 139 {
 140         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 141                 if (wa->address)
 142                         dma_pool_free(wa->dma_pool, wa->address,
 143                                       wa->dma.address);
 144         } else {
 145                 if (wa->dma.address)
 146                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 147                                          wa->dma.dir);
 148                 kfree(wa->address);
 149         }
 150
 151         wa->address = NULL;
 152         wa->dma.address = 0;
 153 }
 154
 155 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 156                                 struct ccp_cmd_queue *cmd_q,
 157                                 unsigned int len,
 158                                 enum dma_data_direction dir)
 159 {
 160         memset(wa, 0, sizeof(*wa));
 161
 162         if (!len)
 163                 return 0;
 164
 165         wa->dev = cmd_q->ccp->dev;
 166         wa->length = len;
 167
 168         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 169                 wa->dma_pool = cmd_q->dma_pool;
 170
 171                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 172                                              &wa->dma.address);
 173                 if (!wa->address)
 174                         return -ENOMEM;
 175
 176                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 177
 178                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 179         } else {
 180                 wa->address = kzalloc(len, GFP_KERNEL);
 181                 if (!wa->address)
 182                         return -ENOMEM;
 183
 184                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 185                                                  dir);
 186                 if (dma_mapping_error(wa->dev, wa->dma.address))
 187                         return -ENOMEM;
 188
 189                 wa->dma.length = len;
 190         }
 191         wa->dma.dir = dir;
 192
 193         return 0;
 194 }
 195
 196 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 197                            struct scatterlist *sg, unsigned int sg_offset,
 198                            unsigned int len)
 199 {
 200         WARN_ON(!wa->address);
 201
 202         if (len > (wa->length - wa_offset))
 203                 return -EINVAL;
 204
 205         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 206                                  0);
 207         return 0;
 208 }
 209
 210 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 211                             struct scatterlist *sg, unsigned int sg_offset,
 212                             unsigned int len)
 213 {
 214         WARN_ON(!wa->address);
 215
 216         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 217                                  1);
 218 }
 219
 220 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 221                                    unsigned int wa_offset,
 222                                    struct scatterlist *sg,
 223                                    unsigned int sg_offset,
 224                                    unsigned int len)
 225 {
 226         u8 *p, *q;
 227         int     rc;
 228
 229         rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
 230         if (rc)
 231                 return rc;
 232
 233         p = wa->address + wa_offset;
 234         q = p + len - 1;
 235         while (p < q) {
 236                 *p = *p ^ *q;
 237                 *q = *p ^ *q;
 238                 *p = *p ^ *q;
 239                 p++;
 240                 q--;
 241         }
 242         return 0;
 243 }
 244
 245 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 246                                     unsigned int wa_offset,
 247                                     struct scatterlist *sg,
 248                                     unsigned int sg_offset,
 249                                     unsigned int len)
 250 {
 251         u8 *p, *q;
 252
 253         p = wa->address + wa_offset;
 254         q = p + len - 1;
 255         while (p < q) {
 256                 *p = *p ^ *q;
 257                 *q = *p ^ *q;
 258                 *p = *p ^ *q;
 259                 p++;
 260                 q--;
 261         }
 262
 263         ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
 264 }
 265
 266 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 267 {
 268         ccp_dm_free(&data->dm_wa);
 269         ccp_sg_free(&data->sg_wa);
 270 }
 271
 272 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 273                          struct scatterlist *sg, u64 sg_len,
 274                          unsigned int dm_len,
 275                          enum dma_data_direction dir)
 276 {
 277         int ret;
 278
 279         memset(data, 0, sizeof(*data));
 280
 281         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 282                                    dir);
 283         if (ret)
 284                 goto e_err;
 285
 286         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 287         if (ret)
 288                 goto e_err;
 289
 290         return 0;
 291
 292 e_err:
 293         ccp_free_data(data, cmd_q);
 294
 295         return ret;
 296 }
 297
 298 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 299 {
 300         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 301         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 302         unsigned int buf_count, nbytes;
 303
 304         /* Clear the buffer if setting it */
 305         if (!from)
 306                 memset(dm_wa->address, 0, dm_wa->length);
 307
 308         if (!sg_wa->sg)
 309                 return 0;
 310
 311         /* Perform the copy operation
 312          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 313          *   an unsigned int
 314          */
 315         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 316         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 317                                  nbytes, from);
 318
 319         /* Update the structures and generate the count */
 320         buf_count = 0;
 321         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 322                 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
 323                              dm_wa->length - buf_count);
 324                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 325
 326                 buf_count += nbytes;
 327                 ccp_update_sg_workarea(sg_wa, nbytes);
 328         }
 329
 330         return buf_count;
 331 }
 332
 333 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 334 {
 335         return ccp_queue_buf(data, 0);
 336 }
 337
 338 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 339 {
 340         return ccp_queue_buf(data, 1);
 341 }
 342
 343 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 344                              struct ccp_op *op, unsigned int block_size,
 345                              bool blocksize_op)
 346 {
 347         unsigned int sg_src_len, sg_dst_len, op_len;
 348
 349         /* The CCP can only DMA from/to one address each per operation. This
 350          * requires that we find the smallest DMA area between the source
 351          * and destination. The resulting len values will always be <= UINT_MAX
 352          * because the dma length is an unsigned int.
 353          */
 354         sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
 355         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 356
 357         if (dst) {
 358                 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
 359                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 360                 op_len = min(sg_src_len, sg_dst_len);
 361         } else {
 362                 op_len = sg_src_len;
 363         }
 364
 365         /* The data operation length will be at least block_size in length
 366          * or the smaller of available sg room remaining for the source or
 367          * the destination
 368          */
 369         op_len = max(op_len, block_size);
 370
 371         /* Unless we have to buffer data, there's no reason to wait */
 372         op->soc = 0;
 373
 374         if (sg_src_len < block_size) {
 375                 /* Not enough data in the sg element, so it
 376                  * needs to be buffered into a blocksize chunk
 377                  */
 378                 int cp_len = ccp_fill_queue_buf(src);
 379
 380                 op->soc = 1;
 381                 op->src.u.dma.address = src->dm_wa.dma.address;
 382                 op->src.u.dma.offset = 0;
 383                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 384         } else {
 385                 /* Enough data in the sg element, but we need to
 386                  * adjust for any previously copied data
 387                  */
 388                 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
 389                 op->src.u.dma.offset = src->sg_wa.sg_used;
 390                 op->src.u.dma.length = op_len & ~(block_size - 1);
 391
 392                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 393         }
 394
 395         if (dst) {
 396                 if (sg_dst_len < block_size) {
 397                         /* Not enough room in the sg element or we're on the
 398                          * last piece of data (when using padding), so the
 399                          * output needs to be buffered into a blocksize chunk
 400                          */
 401                         op->soc = 1;
 402                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 403                         op->dst.u.dma.offset = 0;
 404                         op->dst.u.dma.length = op->src.u.dma.length;
 405                 } else {
 406                         /* Enough room in the sg element, but we need to
 407                          * adjust for any previously used area
 408                          */
 409                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
 410                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 411                         op->dst.u.dma.length = op->src.u.dma.length;
 412                 }
 413         }
 414 }
 415
 416 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 417                              struct ccp_op *op)
 418 {
 419         op->init = 0;
 420
 421         if (dst) {
 422                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 423                         ccp_empty_queue_buf(dst);
 424                 else
 425                         ccp_update_sg_workarea(&dst->sg_wa,
 426                                                op->dst.u.dma.length);
 427         }
 428 }
 429
 430 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 431                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 432                                u32 byte_swap, bool from)
 433 {
 434         struct ccp_op op;
 435
 436         memset(&op, 0, sizeof(op));
 437
 438         op.cmd_q = cmd_q;
 439         op.jobid = jobid;
 440         op.eom = 1;
 441
 442         if (from) {
 443                 op.soc = 1;
 444                 op.src.type = CCP_MEMTYPE_SB;
 445                 op.src.u.sb = sb;
 446                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 447                 op.dst.u.dma.address = wa->dma.address;
 448                 op.dst.u.dma.length = wa->length;
 449         } else {
 450                 op.src.type = CCP_MEMTYPE_SYSTEM;
 451                 op.src.u.dma.address = wa->dma.address;
 452                 op.src.u.dma.length = wa->length;
 453                 op.dst.type = CCP_MEMTYPE_SB;
 454                 op.dst.u.sb = sb;
 455         }
 456
 457         op.u.passthru.byte_swap = byte_swap;
 458
 459         return cmd_q->ccp->vdata->perform->passthru(&op);
 460 }
 461
 462 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 463                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 464                           u32 byte_swap)
 465 {
 466         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 467 }
 468
 469 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 470                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 471                             u32 byte_swap)
 472 {
 473         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 474 }
 475
 476 static noinline_for_stack int
 477 ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 478 {
 479         struct ccp_aes_engine *aes = &cmd->u.aes;
 480         struct ccp_dm_workarea key, ctx;
 481         struct ccp_data src;
 482         struct ccp_op op;
 483         unsigned int dm_offset;
 484         int ret;
 485
 486         if (!((aes->key_len == AES_KEYSIZE_128) ||
 487               (aes->key_len == AES_KEYSIZE_192) ||
 488               (aes->key_len == AES_KEYSIZE_256)))
 489                 return -EINVAL;
 490
 491         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 492                 return -EINVAL;
 493
 494         if (aes->iv_len != AES_BLOCK_SIZE)
 495                 return -EINVAL;
 496
 497         if (!aes->key || !aes->iv || !aes->src)
 498                 return -EINVAL;
 499
 500         if (aes->cmac_final) {
 501                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 502                         return -EINVAL;
 503
 504                 if (!aes->cmac_key)
 505                         return -EINVAL;
 506         }
 507
 508         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 509         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 510
 511         ret = -EIO;
 512         memset(&op, 0, sizeof(op));
 513         op.cmd_q = cmd_q;
 514         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 515         op.sb_key = cmd_q->sb_key;
 516         op.sb_ctx = cmd_q->sb_ctx;
 517         op.init = 1;
 518         op.u.aes.type = aes->type;
 519         op.u.aes.mode = aes->mode;
 520         op.u.aes.action = aes->action;
 521
 522         /* All supported key sizes fit in a single (32-byte) SB entry
 523          * and must be in little endian format. Use the 256-bit byte
 524          * swap passthru option to convert from big endian to little
 525          * endian.
 526          */
 527         ret = ccp_init_dm_workarea(&key, cmd_q,
 528                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 529                                    DMA_TO_DEVICE);
 530         if (ret)
 531                 return ret;
 532
 533         dm_offset = CCP_SB_BYTES - aes->key_len;
 534         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 535         if (ret)
 536                 goto e_key;
 537         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 538                              CCP_PASSTHRU_BYTESWAP_256BIT);
 539         if (ret) {
 540                 cmd->engine_error = cmd_q->cmd_error;
 541                 goto e_key;
 542         }
 543
 544         /* The AES context fits in a single (32-byte) SB entry and
 545          * must be in little endian format. Use the 256-bit byte swap
 546          * passthru option to convert from big endian to little endian.
 547          */
 548         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 549                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 550                                    DMA_BIDIRECTIONAL);
 551         if (ret)
 552                 goto e_key;
 553
 554         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 555         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 556         if (ret)
 557                 goto e_ctx;
 558         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 559                              CCP_PASSTHRU_BYTESWAP_256BIT);
 560         if (ret) {
 561                 cmd->engine_error = cmd_q->cmd_error;
 562                 goto e_ctx;
 563         }
 564
 565         /* Send data to the CCP AES engine */
 566         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 567                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 568         if (ret)
 569                 goto e_ctx;
 570
 571         while (src.sg_wa.bytes_left) {
 572                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 573                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 574                         op.eom = 1;
 575
 576                         /* Push the K1/K2 key to the CCP now */
 577                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 578                                                op.sb_ctx,
 579                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 580                         if (ret) {
 581                                 cmd->engine_error = cmd_q->cmd_error;
 582                                 goto e_src;
 583                         }
 584
 585                         ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 586                                               aes->cmac_key_len);
 587                         if (ret)
 588                                 goto e_src;
 589                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 590                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 591                         if (ret) {
 592                                 cmd->engine_error = cmd_q->cmd_error;
 593                                 goto e_src;
 594                         }
 595                 }
 596
 597                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 598                 if (ret) {
 599                         cmd->engine_error = cmd_q->cmd_error;
 600                         goto e_src;
 601                 }
 602
 603                 ccp_process_data(&src, NULL, &op);
 604         }
 605
 606         /* Retrieve the AES context - convert from LE to BE using
 607          * 32-byte (256-bit) byteswapping
 608          */
 609         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 610                                CCP_PASSTHRU_BYTESWAP_256BIT);
 611         if (ret) {
 612                 cmd->engine_error = cmd_q->cmd_error;
 613                 goto e_src;
 614         }
 615
 616         /* ...but we only need AES_BLOCK_SIZE bytes */
 617         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 618         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 619
 620 e_src:
 621         ccp_free_data(&src, cmd_q);
 622
 623 e_ctx:
 624         ccp_dm_free(&ctx);
 625
 626 e_key:
 627         ccp_dm_free(&key);
 628
 629         return ret;
 630 }
 631
 632 static noinline_for_stack int
 633 ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 634 {
 635         struct ccp_aes_engine *aes = &cmd->u.aes;
 636         struct ccp_dm_workarea key, ctx, final_wa, tag;
 637         struct ccp_data src, dst;
 638         struct ccp_data aad;
 639         struct ccp_op op;
 640
 641         unsigned long long *final;
 642         unsigned int dm_offset;
 643         unsigned int authsize;
 644         unsigned int jobid;
 645         unsigned int ilen;
 646         bool in_place = true; /* Default value */
 647         int ret;
 648
 649         struct scatterlist *p_inp, sg_inp[2];
 650         struct scatterlist *p_tag, sg_tag[2];
 651         struct scatterlist *p_outp, sg_outp[2];
 652         struct scatterlist *p_aad;
 653
 654         if (!aes->iv)
 655                 return -EINVAL;
 656
 657         if (!((aes->key_len == AES_KEYSIZE_128) ||
 658                 (aes->key_len == AES_KEYSIZE_192) ||
 659                 (aes->key_len == AES_KEYSIZE_256)))
 660                 return -EINVAL;
 661
 662         if (!aes->key) /* Gotta have a key SGL */
 663                 return -EINVAL;
 664
 665         /* Zero defaults to 16 bytes, the maximum size */
 666         authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
 667         switch (authsize) {
 668         case 16:
 669         case 15:
 670         case 14:
 671         case 13:
 672         case 12:
 673         case 8:
 674         case 4:
 675                 break;
 676         default:
 677                 return -EINVAL;
 678         }
 679
 680         /* First, decompose the source buffer into AAD & PT,
 681          * and the destination buffer into AAD, CT & tag, or
 682          * the input into CT & tag.
 683          * It is expected that the input and output SGs will
 684          * be valid, even if the AAD and input lengths are 0.
 685          */
 686         p_aad = aes->src;
 687         p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
 688         p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
 689         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 690                 ilen = aes->src_len;
 691                 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
 692         } else {
 693                 /* Input length for decryption includes tag */
 694                 ilen = aes->src_len - authsize;
 695                 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
 696         }
 697
 698         jobid = CCP_NEW_JOBID(cmd_q->ccp);
 699
 700         memset(&op, 0, sizeof(op));
 701         op.cmd_q = cmd_q;
 702         op.jobid = jobid;
 703         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 704         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 705         op.init = 1;
 706         op.u.aes.type = aes->type;
 707
 708         /* Copy the key to the LSB */
 709         ret = ccp_init_dm_workarea(&key, cmd_q,
 710                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 711                                    DMA_TO_DEVICE);
 712         if (ret)
 713                 return ret;
 714
 715         dm_offset = CCP_SB_BYTES - aes->key_len;
 716         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 717         if (ret)
 718                 goto e_key;
 719         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 720                              CCP_PASSTHRU_BYTESWAP_256BIT);
 721         if (ret) {
 722                 cmd->engine_error = cmd_q->cmd_error;
 723                 goto e_key;
 724         }
 725
 726         /* Copy the context (IV) to the LSB.
 727          * There is an assumption here that the IV is 96 bits in length, plus
 728          * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
 729          */
 730         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 731                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 732                                    DMA_BIDIRECTIONAL);
 733         if (ret)
 734                 goto e_key;
 735
 736         dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
 737         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 738         if (ret)
 739                 goto e_ctx;
 740
 741         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 742                              CCP_PASSTHRU_BYTESWAP_256BIT);
 743         if (ret) {
 744                 cmd->engine_error = cmd_q->cmd_error;
 745                 goto e_ctx;
 746         }
 747
 748         op.init = 1;
 749         if (aes->aad_len > 0) {
 750                 /* Step 1: Run a GHASH over the Additional Authenticated Data */
 751                 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
 752                                     AES_BLOCK_SIZE,
 753                                     DMA_TO_DEVICE);
 754                 if (ret)
 755                         goto e_ctx;
 756
 757                 op.u.aes.mode = CCP_AES_MODE_GHASH;
 758                 op.u.aes.action = CCP_AES_GHASHAAD;
 759
 760                 while (aad.sg_wa.bytes_left) {
 761                         ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
 762
 763                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 764                         if (ret) {
 765                                 cmd->engine_error = cmd_q->cmd_error;
 766                                 goto e_aad;
 767                         }
 768
 769                         ccp_process_data(&aad, NULL, &op);
 770                         op.init = 0;
 771                 }
 772         }
 773
 774         op.u.aes.mode = CCP_AES_MODE_GCTR;
 775         op.u.aes.action = aes->action;
 776
 777         if (ilen > 0) {
 778                 /* Step 2: Run a GCTR over the plaintext */
 779                 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
 780
 781                 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
 782                                     AES_BLOCK_SIZE,
 783                                     in_place ? DMA_BIDIRECTIONAL
 784                                              : DMA_TO_DEVICE);
 785                 if (ret)
 786                         goto e_aad;
 787
 788                 if (in_place) {
 789                         dst = src;
 790                 } else {
 791                         ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
 792                                             AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 793                         if (ret)
 794                                 goto e_src;
 795                 }
 796
 797                 op.soc = 0;
 798                 op.eom = 0;
 799                 op.init = 1;
 800                 while (src.sg_wa.bytes_left) {
 801                         ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 802                         if (!src.sg_wa.bytes_left) {
 803                                 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
 804
 805                                 if (nbytes) {
 806                                         op.eom = 1;
 807                                         op.u.aes.size = (nbytes * 8) - 1;
 808                                 }
 809                         }
 810
 811                         ret = cmd_q->ccp->vdata->perform->aes(&op);
 812                         if (ret) {
 813                                 cmd->engine_error = cmd_q->cmd_error;
 814                                 goto e_dst;
 815                         }
 816
 817                         ccp_process_data(&src, &dst, &op);
 818                         op.init = 0;
 819                 }
 820         }
 821
 822         /* Step 3: Update the IV portion of the context with the original IV */
 823         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 824                                CCP_PASSTHRU_BYTESWAP_256BIT);
 825         if (ret) {
 826                 cmd->engine_error = cmd_q->cmd_error;
 827                 goto e_dst;
 828         }
 829
 830         ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 831         if (ret)
 832                 goto e_dst;
 833
 834         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 835                              CCP_PASSTHRU_BYTESWAP_256BIT);
 836         if (ret) {
 837                 cmd->engine_error = cmd_q->cmd_error;
 838                 goto e_dst;
 839         }
 840
 841         /* Step 4: Concatenate the lengths of the AAD and source, and
 842          * hash that 16 byte buffer.
 843          */
 844         ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
 845                                    DMA_BIDIRECTIONAL);
 846         if (ret)
 847                 goto e_dst;
 848         final = (unsigned long long *) final_wa.address;
 849         final[0] = cpu_to_be64(aes->aad_len * 8);
 850         final[1] = cpu_to_be64(ilen * 8);
 851
 852         memset(&op, 0, sizeof(op));
 853         op.cmd_q = cmd_q;
 854         op.jobid = jobid;
 855         op.sb_key = cmd_q->sb_key; /* Pre-allocated */
 856         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
 857         op.init = 1;
 858         op.u.aes.type = aes->type;
 859         op.u.aes.mode = CCP_AES_MODE_GHASH;
 860         op.u.aes.action = CCP_AES_GHASHFINAL;
 861         op.src.type = CCP_MEMTYPE_SYSTEM;
 862         op.src.u.dma.address = final_wa.dma.address;
 863         op.src.u.dma.length = AES_BLOCK_SIZE;
 864         op.dst.type = CCP_MEMTYPE_SYSTEM;
 865         op.dst.u.dma.address = final_wa.dma.address;
 866         op.dst.u.dma.length = AES_BLOCK_SIZE;
 867         op.eom = 1;
 868         op.u.aes.size = 0;
 869         ret = cmd_q->ccp->vdata->perform->aes(&op);
 870         if (ret)
 871                 goto e_final_wa;
 872
 873         if (aes->action == CCP_AES_ACTION_ENCRYPT) {
 874                 /* Put the ciphered tag after the ciphertext. */
 875                 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
 876         } else {
 877                 /* Does this ciphered tag match the input? */
 878                 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
 879                                            DMA_BIDIRECTIONAL);
 880                 if (ret)
 881                         goto e_final_wa;
 882                 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
 883                 if (ret) {
 884                         ccp_dm_free(&tag);
 885                         goto e_final_wa;
 886                 }
 887
 888                 ret = crypto_memneq(tag.address, final_wa.address,
 889                                     authsize) ? -EBADMSG : 0;
 890                 ccp_dm_free(&tag);
 891         }
 892
 893 e_final_wa:
 894         ccp_dm_free(&final_wa);
 895
 896 e_dst:
 897         if (ilen > 0 && !in_place)
 898                 ccp_free_data(&dst, cmd_q);
 899
 900 e_src:
 901         if (ilen > 0)
 902                 ccp_free_data(&src, cmd_q);
 903
 904 e_aad:
 905         if (aes->aad_len)
 906                 ccp_free_data(&aad, cmd_q);
 907
 908 e_ctx:
 909         ccp_dm_free(&ctx);
 910
 911 e_key:
 912         ccp_dm_free(&key);
 913
 914         return ret;
 915 }
 916
 917 static noinline_for_stack int
 918 ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 919 {
 920         struct ccp_aes_engine *aes = &cmd->u.aes;
 921         struct ccp_dm_workarea key, ctx;
 922         struct ccp_data src, dst;
 923         struct ccp_op op;
 924         unsigned int dm_offset;
 925         bool in_place = false;
 926         int ret;
 927
 928         if (!((aes->key_len == AES_KEYSIZE_128) ||
 929               (aes->key_len == AES_KEYSIZE_192) ||
 930               (aes->key_len == AES_KEYSIZE_256)))
 931                 return -EINVAL;
 932
 933         if (((aes->mode == CCP_AES_MODE_ECB) ||
 934              (aes->mode == CCP_AES_MODE_CBC) ||
 935              (aes->mode == CCP_AES_MODE_CFB)) &&
 936             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 937                 return -EINVAL;
 938
 939         if (!aes->key || !aes->src || !aes->dst)
 940                 return -EINVAL;
 941
 942         if (aes->mode != CCP_AES_MODE_ECB) {
 943                 if (aes->iv_len != AES_BLOCK_SIZE)
 944                         return -EINVAL;
 945
 946                 if (!aes->iv)
 947                         return -EINVAL;
 948         }
 949
 950         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 951         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 952
 953         ret = -EIO;
 954         memset(&op, 0, sizeof(op));
 955         op.cmd_q = cmd_q;
 956         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 957         op.sb_key = cmd_q->sb_key;
 958         op.sb_ctx = cmd_q->sb_ctx;
 959         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 960         op.u.aes.type = aes->type;
 961         op.u.aes.mode = aes->mode;
 962         op.u.aes.action = aes->action;
 963
 964         /* All supported key sizes fit in a single (32-byte) SB entry
 965          * and must be in little endian format. Use the 256-bit byte
 966          * swap passthru option to convert from big endian to little
 967          * endian.
 968          */
 969         ret = ccp_init_dm_workarea(&key, cmd_q,
 970                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 971                                    DMA_TO_DEVICE);
 972         if (ret)
 973                 return ret;
 974
 975         dm_offset = CCP_SB_BYTES - aes->key_len;
 976         ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 977         if (ret)
 978                 goto e_key;
 979         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 980                              CCP_PASSTHRU_BYTESWAP_256BIT);
 981         if (ret) {
 982                 cmd->engine_error = cmd_q->cmd_error;
 983                 goto e_key;
 984         }
 985
 986         /* The AES context fits in a single (32-byte) SB entry and
 987          * must be in little endian format. Use the 256-bit byte swap
 988          * passthru option to convert from big endian to little endian.
 989          */
 990         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 991                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 992                                    DMA_BIDIRECTIONAL);
 993         if (ret)
 994                 goto e_key;
 995
 996         if (aes->mode != CCP_AES_MODE_ECB) {
 997                 /* Load the AES context - convert to LE */
 998                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 999                 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1000                 if (ret)
1001                         goto e_ctx;
1002                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1003                                      CCP_PASSTHRU_BYTESWAP_256BIT);
1004                 if (ret) {
1005                         cmd->engine_error = cmd_q->cmd_error;
1006                         goto e_ctx;
1007                 }
1008         }
1009         switch (aes->mode) {
1010         case CCP_AES_MODE_CFB: /* CFB128 only */
1011         case CCP_AES_MODE_CTR:
1012                 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1013                 break;
1014         default:
1015                 op.u.aes.size = 0;
1016         }
1017
1018         /* Prepare the input and output data workareas. For in-place
1019          * operations we need to set the dma direction to BIDIRECTIONAL
1020          * and copy the src workarea to the dst workarea.
1021          */
1022         if (sg_virt(aes->src) == sg_virt(aes->dst))
1023                 in_place = true;
1024
1025         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1026                             AES_BLOCK_SIZE,
1027                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1028         if (ret)
1029                 goto e_ctx;
1030
1031         if (in_place) {
1032                 dst = src;
1033         } else {
1034                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1035                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1036                 if (ret)
1037                         goto e_src;
1038         }
1039
1040         /* Send data to the CCP AES engine */
1041         while (src.sg_wa.bytes_left) {
1042                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1043                 if (!src.sg_wa.bytes_left) {
1044                         op.eom = 1;
1045
1046                         /* Since we don't retrieve the AES context in ECB
1047                          * mode we have to wait for the operation to complete
1048                          * on the last piece of data
1049                          */
1050                         if (aes->mode == CCP_AES_MODE_ECB)
1051                                 op.soc = 1;
1052                 }
1053
1054                 ret = cmd_q->ccp->vdata->perform->aes(&op);
1055                 if (ret) {
1056                         cmd->engine_error = cmd_q->cmd_error;
1057                         goto e_dst;
1058                 }
1059
1060                 ccp_process_data(&src, &dst, &op);
1061         }
1062
1063         if (aes->mode != CCP_AES_MODE_ECB) {
1064                 /* Retrieve the AES context - convert from LE to BE using
1065                  * 32-byte (256-bit) byteswapping
1066                  */
1067                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1068                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1069                 if (ret) {
1070                         cmd->engine_error = cmd_q->cmd_error;
1071                         goto e_dst;
1072                 }
1073
1074                 /* ...but we only need AES_BLOCK_SIZE bytes */
1075                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1076                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1077         }
1078
1079 e_dst:
1080         if (!in_place)
1081                 ccp_free_data(&dst, cmd_q);
1082
1083 e_src:
1084         ccp_free_data(&src, cmd_q);
1085
1086 e_ctx:
1087         ccp_dm_free(&ctx);
1088
1089 e_key:
1090         ccp_dm_free(&key);
1091
1092         return ret;
1093 }
1094
1095 static noinline_for_stack int
1096 ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1097 {
1098         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1099         struct ccp_dm_workarea key, ctx;
1100         struct ccp_data src, dst;
1101         struct ccp_op op;
1102         unsigned int unit_size, dm_offset;
1103         bool in_place = false;
1104         unsigned int sb_count;
1105         enum ccp_aes_type aestype;
1106         int ret;
1107
1108         switch (xts->unit_size) {
1109         case CCP_XTS_AES_UNIT_SIZE_16:
1110                 unit_size = 16;
1111                 break;
1112         case CCP_XTS_AES_UNIT_SIZE_512:
1113                 unit_size = 512;
1114                 break;
1115         case CCP_XTS_AES_UNIT_SIZE_1024:
1116                 unit_size = 1024;
1117                 break;
1118         case CCP_XTS_AES_UNIT_SIZE_2048:
1119                 unit_size = 2048;
1120                 break;
1121         case CCP_XTS_AES_UNIT_SIZE_4096:
1122                 unit_size = 4096;
1123                 break;
1124
1125         default:
1126                 return -EINVAL;
1127         }
1128
1129         if (xts->key_len == AES_KEYSIZE_128)
1130                 aestype = CCP_AES_TYPE_128;
1131         else if (xts->key_len == AES_KEYSIZE_256)
1132                 aestype = CCP_AES_TYPE_256;
1133         else
1134                 return -EINVAL;
1135
1136         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1137                 return -EINVAL;
1138
1139         if (xts->iv_len != AES_BLOCK_SIZE)
1140                 return -EINVAL;
1141
1142         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1143                 return -EINVAL;
1144
1145         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1146         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1147
1148         ret = -EIO;
1149         memset(&op, 0, sizeof(op));
1150         op.cmd_q = cmd_q;
1151         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1152         op.sb_key = cmd_q->sb_key;
1153         op.sb_ctx = cmd_q->sb_ctx;
1154         op.init = 1;
1155         op.u.xts.type = aestype;
1156         op.u.xts.action = xts->action;
1157         op.u.xts.unit_size = xts->unit_size;
1158
1159         /* A version 3 device only supports 128-bit keys, which fits into a
1160          * single SB entry. A version 5 device uses a 512-bit vector, so two
1161          * SB entries.
1162          */
1163         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1164                 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1165         else
1166                 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1167         ret = ccp_init_dm_workarea(&key, cmd_q,
1168                                    sb_count * CCP_SB_BYTES,
1169                                    DMA_TO_DEVICE);
1170         if (ret)
1171                 return ret;
1172
1173         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1174                 /* All supported key sizes must be in little endian format.
1175                  * Use the 256-bit byte swap passthru option to convert from
1176                  * big endian to little endian.
1177                  */
1178                 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1179                 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1180                 if (ret)
1181                         goto e_key;
1182                 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1183                 if (ret)
1184                         goto e_key;
1185         } else {
1186                 /* Version 5 CCPs use a 512-bit space for the key: each portion
1187                  * occupies 256 bits, or one entire slot, and is zero-padded.
1188                  */
1189                 unsigned int pad;
1190
1191                 dm_offset = CCP_SB_BYTES;
1192                 pad = dm_offset - xts->key_len;
1193                 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1194                 if (ret)
1195                         goto e_key;
1196                 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1197                                       xts->key_len, xts->key_len);
1198                 if (ret)
1199                         goto e_key;
1200         }
1201         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1202                              CCP_PASSTHRU_BYTESWAP_256BIT);
1203         if (ret) {
1204                 cmd->engine_error = cmd_q->cmd_error;
1205                 goto e_key;
1206         }
1207
1208         /* The AES context fits in a single (32-byte) SB entry and
1209          * for XTS is already in little endian format so no byte swapping
1210          * is needed.
1211          */
1212         ret = ccp_init_dm_workarea(&ctx, cmd_q,
1213                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1214                                    DMA_BIDIRECTIONAL);
1215         if (ret)
1216                 goto e_key;
1217
1218         ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1219         if (ret)
1220                 goto e_ctx;
1221         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1222                              CCP_PASSTHRU_BYTESWAP_NOOP);
1223         if (ret) {
1224                 cmd->engine_error = cmd_q->cmd_error;
1225                 goto e_ctx;
1226         }
1227
1228         /* Prepare the input and output data workareas. For in-place
1229          * operations we need to set the dma direction to BIDIRECTIONAL
1230          * and copy the src workarea to the dst workarea.
1231          */
1232         if (sg_virt(xts->src) == sg_virt(xts->dst))
1233                 in_place = true;
1234
1235         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1236                             unit_size,
1237                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1238         if (ret)
1239                 goto e_ctx;
1240
1241         if (in_place) {
1242                 dst = src;
1243         } else {
1244                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1245                                     unit_size, DMA_FROM_DEVICE);
1246                 if (ret)
1247                         goto e_src;
1248         }
1249
1250         /* Send data to the CCP AES engine */
1251         while (src.sg_wa.bytes_left) {
1252                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1253                 if (!src.sg_wa.bytes_left)
1254                         op.eom = 1;
1255
1256                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1257                 if (ret) {
1258                         cmd->engine_error = cmd_q->cmd_error;
1259                         goto e_dst;
1260                 }
1261
1262                 ccp_process_data(&src, &dst, &op);
1263         }
1264
1265         /* Retrieve the AES context - convert from LE to BE using
1266          * 32-byte (256-bit) byteswapping
1267          */
1268         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1269                                CCP_PASSTHRU_BYTESWAP_256BIT);
1270         if (ret) {
1271                 cmd->engine_error = cmd_q->cmd_error;
1272                 goto e_dst;
1273         }
1274
1275         /* ...but we only need AES_BLOCK_SIZE bytes */
1276         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1277         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1278
1279 e_dst:
1280         if (!in_place)
1281                 ccp_free_data(&dst, cmd_q);
1282
1283 e_src:
1284         ccp_free_data(&src, cmd_q);
1285
1286 e_ctx:
1287         ccp_dm_free(&ctx);
1288
1289 e_key:
1290         ccp_dm_free(&key);
1291
1292         return ret;
1293 }
1294
1295 static noinline_for_stack int
1296 ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1297 {
1298         struct ccp_des3_engine *des3 = &cmd->u.des3;
1299
1300         struct ccp_dm_workarea key, ctx;
1301         struct ccp_data src, dst;
1302         struct ccp_op op;
1303         unsigned int dm_offset;
1304         unsigned int len_singlekey;
1305         bool in_place = false;
1306         int ret;
1307
1308         /* Error checks */
1309         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1310                 return -EINVAL;
1311
1312         if (!cmd_q->ccp->vdata->perform->des3)
1313                 return -EINVAL;
1314
1315         if (des3->key_len != DES3_EDE_KEY_SIZE)
1316                 return -EINVAL;
1317
1318         if (((des3->mode == CCP_DES3_MODE_ECB) ||
1319                 (des3->mode == CCP_DES3_MODE_CBC)) &&
1320                 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1321                 return -EINVAL;
1322
1323         if (!des3->key || !des3->src || !des3->dst)
1324                 return -EINVAL;
1325
1326         if (des3->mode != CCP_DES3_MODE_ECB) {
1327                 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1328                         return -EINVAL;
1329
1330                 if (!des3->iv)
1331                         return -EINVAL;
1332         }
1333
1334         ret = -EIO;
1335         /* Zero out all the fields of the command desc */
1336         memset(&op, 0, sizeof(op));
1337
1338         /* Set up the Function field */
1339         op.cmd_q = cmd_q;
1340         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1341         op.sb_key = cmd_q->sb_key;
1342
1343         op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1344         op.u.des3.type = des3->type;
1345         op.u.des3.mode = des3->mode;
1346         op.u.des3.action = des3->action;
1347
1348         /*
1349          * All supported key sizes fit in a single (32-byte) KSB entry and
1350          * (like AES) must be in little endian format. Use the 256-bit byte
1351          * swap passthru option to convert from big endian to little endian.
1352          */
1353         ret = ccp_init_dm_workarea(&key, cmd_q,
1354                                    CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1355                                    DMA_TO_DEVICE);
1356         if (ret)
1357                 return ret;
1358
1359         /*
1360          * The contents of the key triplet are in the reverse order of what
1361          * is required by the engine. Copy the 3 pieces individually to put
1362          * them where they belong.
1363          */
1364         dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1365
1366         len_singlekey = des3->key_len / 3;
1367         ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1368                               des3->key, 0, len_singlekey);
1369         if (ret)
1370                 goto e_key;
1371         ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1372                               des3->key, len_singlekey, len_singlekey);
1373         if (ret)
1374                 goto e_key;
1375         ret = ccp_set_dm_area(&key, dm_offset,
1376                               des3->key, 2 * len_singlekey, len_singlekey);
1377         if (ret)
1378                 goto e_key;
1379
1380         /* Copy the key to the SB */
1381         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1382                              CCP_PASSTHRU_BYTESWAP_256BIT);
1383         if (ret) {
1384                 cmd->engine_error = cmd_q->cmd_error;
1385                 goto e_key;
1386         }
1387
1388         /*
1389          * The DES3 context fits in a single (32-byte) KSB entry and
1390          * must be in little endian format. Use the 256-bit byte swap
1391          * passthru option to convert from big endian to little endian.
1392          */
1393         if (des3->mode != CCP_DES3_MODE_ECB) {
1394                 op.sb_ctx = cmd_q->sb_ctx;
1395
1396                 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1397                                            CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1398                                            DMA_BIDIRECTIONAL);
1399                 if (ret)
1400                         goto e_key;
1401
1402                 /* Load the context into the LSB */
1403                 dm_offset = CCP_SB_BYTES - des3->iv_len;
1404                 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1405                                       des3->iv_len);
1406                 if (ret)
1407                         goto e_ctx;
1408
1409                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1410                                      CCP_PASSTHRU_BYTESWAP_256BIT);
1411                 if (ret) {
1412                         cmd->engine_error = cmd_q->cmd_error;
1413                         goto e_ctx;
1414                 }
1415         }
1416
1417         /*
1418          * Prepare the input and output data workareas. For in-place
1419          * operations we need to set the dma direction to BIDIRECTIONAL
1420          * and copy the src workarea to the dst workarea.
1421          */
1422         if (sg_virt(des3->src) == sg_virt(des3->dst))
1423                 in_place = true;
1424
1425         ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1426                         DES3_EDE_BLOCK_SIZE,
1427                         in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1428         if (ret)
1429                 goto e_ctx;
1430
1431         if (in_place)
1432                 dst = src;
1433         else {
1434                 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1435                                 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1436                 if (ret)
1437                         goto e_src;
1438         }
1439
1440         /* Send data to the CCP DES3 engine */
1441         while (src.sg_wa.bytes_left) {
1442                 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1443                 if (!src.sg_wa.bytes_left) {
1444                         op.eom = 1;
1445
1446                         /* Since we don't retrieve the context in ECB mode
1447                          * we have to wait for the operation to complete
1448                          * on the last piece of data
1449                          */
1450                         op.soc = 0;
1451                 }
1452
1453                 ret = cmd_q->ccp->vdata->perform->des3(&op);
1454                 if (ret) {
1455                         cmd->engine_error = cmd_q->cmd_error;
1456                         goto e_dst;
1457                 }
1458
1459                 ccp_process_data(&src, &dst, &op);
1460         }
1461
1462         if (des3->mode != CCP_DES3_MODE_ECB) {
1463                 /* Retrieve the context and make BE */
1464                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1465                                        CCP_PASSTHRU_BYTESWAP_256BIT);
1466                 if (ret) {
1467                         cmd->engine_error = cmd_q->cmd_error;
1468                         goto e_dst;
1469                 }
1470
1471                 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1472                 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1473                                 DES3_EDE_BLOCK_SIZE);
1474         }
1475 e_dst:
1476         if (!in_place)
1477                 ccp_free_data(&dst, cmd_q);
1478
1479 e_src:
1480         ccp_free_data(&src, cmd_q);
1481
1482 e_ctx:
1483         if (des3->mode != CCP_DES3_MODE_ECB)
1484                 ccp_dm_free(&ctx);
1485
1486 e_key:
1487         ccp_dm_free(&key);
1488
1489         return ret;
1490 }
1491
1492 static noinline_for_stack int
1493 ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1494 {
1495         struct ccp_sha_engine *sha = &cmd->u.sha;
1496         struct ccp_dm_workarea ctx;
1497         struct ccp_data src;
1498         struct ccp_op op;
1499         unsigned int ioffset, ooffset;
1500         unsigned int digest_size;
1501         int sb_count;
1502         const void *init;
1503         u64 block_size;
1504         int ctx_size;
1505         int ret;
1506
1507         switch (sha->type) {
1508         case CCP_SHA_TYPE_1:
1509                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1510                         return -EINVAL;
1511                 block_size = SHA1_BLOCK_SIZE;
1512                 break;
1513         case CCP_SHA_TYPE_224:
1514                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1515                         return -EINVAL;
1516                 block_size = SHA224_BLOCK_SIZE;
1517                 break;
1518         case CCP_SHA_TYPE_256:
1519                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1520                         return -EINVAL;
1521                 block_size = SHA256_BLOCK_SIZE;
1522                 break;
1523         case CCP_SHA_TYPE_384:
1524                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1525                     || sha->ctx_len < SHA384_DIGEST_SIZE)
1526                         return -EINVAL;
1527                 block_size = SHA384_BLOCK_SIZE;
1528                 break;
1529         case CCP_SHA_TYPE_512:
1530                 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1531                     || sha->ctx_len < SHA512_DIGEST_SIZE)
1532                         return -EINVAL;
1533                 block_size = SHA512_BLOCK_SIZE;
1534                 break;
1535         default:
1536                 return -EINVAL;
1537         }
1538
1539         if (!sha->ctx)
1540                 return -EINVAL;
1541
1542         if (!sha->final && (sha->src_len & (block_size - 1)))
1543                 return -EINVAL;
1544
1545         /* The version 3 device can't handle zero-length input */
1546         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1547
1548                 if (!sha->src_len) {
1549                         unsigned int digest_len;
1550                         const u8 *sha_zero;
1551
1552                         /* Not final, just return */
1553                         if (!sha->final)
1554                                 return 0;
1555
1556                         /* CCP can't do a zero length sha operation so the
1557                          * caller must buffer the data.
1558                          */
1559                         if (sha->msg_bits)
1560                                 return -EINVAL;
1561
1562                         /* The CCP cannot perform zero-length sha operations
1563                          * so the caller is required to buffer data for the
1564                          * final operation. However, a sha operation for a
1565                          * message with a total length of zero is valid so
1566                          * known values are required to supply the result.
1567                          */
1568                         switch (sha->type) {
1569                         case CCP_SHA_TYPE_1:
1570                                 sha_zero = sha1_zero_message_hash;
1571                                 digest_len = SHA1_DIGEST_SIZE;
1572                                 break;
1573                         case CCP_SHA_TYPE_224:
1574                                 sha_zero = sha224_zero_message_hash;
1575                                 digest_len = SHA224_DIGEST_SIZE;
1576                                 break;
1577                         case CCP_SHA_TYPE_256:
1578                                 sha_zero = sha256_zero_message_hash;
1579                                 digest_len = SHA256_DIGEST_SIZE;
1580                                 break;
1581                         default:
1582                                 return -EINVAL;
1583                         }
1584
1585                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1586                                                  digest_len, 1);
1587
1588                         return 0;
1589                 }
1590         }
1591
1592         /* Set variables used throughout */
1593         switch (sha->type) {
1594         case CCP_SHA_TYPE_1:
1595                 digest_size = SHA1_DIGEST_SIZE;
1596                 init = (void *) ccp_sha1_init;
1597                 ctx_size = SHA1_DIGEST_SIZE;
1598                 sb_count = 1;
1599                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1600                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1601                 else
1602                         ooffset = ioffset = 0;
1603                 break;
1604         case CCP_SHA_TYPE_224:
1605                 digest_size = SHA224_DIGEST_SIZE;
1606                 init = (void *) ccp_sha224_init;
1607                 ctx_size = SHA256_DIGEST_SIZE;
1608                 sb_count = 1;
1609                 ioffset = 0;
1610                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1611                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1612                 else
1613                         ooffset = 0;
1614                 break;
1615         case CCP_SHA_TYPE_256:
1616                 digest_size = SHA256_DIGEST_SIZE;
1617                 init = (void *) ccp_sha256_init;
1618                 ctx_size = SHA256_DIGEST_SIZE;
1619                 sb_count = 1;
1620                 ooffset = ioffset = 0;
1621                 break;
1622         case CCP_SHA_TYPE_384:
1623                 digest_size = SHA384_DIGEST_SIZE;
1624                 init = (void *) ccp_sha384_init;
1625                 ctx_size = SHA512_DIGEST_SIZE;
1626                 sb_count = 2;
1627                 ioffset = 0;
1628                 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1629                 break;
1630         case CCP_SHA_TYPE_512:
1631                 digest_size = SHA512_DIGEST_SIZE;
1632                 init = (void *) ccp_sha512_init;
1633                 ctx_size = SHA512_DIGEST_SIZE;
1634                 sb_count = 2;
1635                 ooffset = ioffset = 0;
1636                 break;
1637         default:
1638                 ret = -EINVAL;
1639                 goto e_data;
1640         }
1641
1642         /* For zero-length plaintext the src pointer is ignored;
1643          * otherwise both parts must be valid
1644          */
1645         if (sha->src_len && !sha->src)
1646                 return -EINVAL;
1647
1648         memset(&op, 0, sizeof(op));
1649         op.cmd_q = cmd_q;
1650         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1651         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1652         op.u.sha.type = sha->type;
1653         op.u.sha.msg_bits = sha->msg_bits;
1654
1655         /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1656          * SHA384/512 require 2 adjacent SB slots, with the right half in the
1657          * first slot, and the left half in the second. Each portion must then
1658          * be in little endian format: use the 256-bit byte swap option.
1659          */
1660         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1661                                    DMA_BIDIRECTIONAL);
1662         if (ret)
1663                 return ret;
1664         if (sha->first) {
1665                 switch (sha->type) {
1666                 case CCP_SHA_TYPE_1:
1667                 case CCP_SHA_TYPE_224:
1668                 case CCP_SHA_TYPE_256:
1669                         memcpy(ctx.address + ioffset, init, ctx_size);
1670                         break;
1671                 case CCP_SHA_TYPE_384:
1672                 case CCP_SHA_TYPE_512:
1673                         memcpy(ctx.address + ctx_size / 2, init,
1674                                ctx_size / 2);
1675                         memcpy(ctx.address, init + ctx_size / 2,
1676                                ctx_size / 2);
1677                         break;
1678                 default:
1679                         ret = -EINVAL;
1680                         goto e_ctx;
1681                 }
1682         } else {
1683                 /* Restore the context */
1684                 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1685                                       sb_count * CCP_SB_BYTES);
1686                 if (ret)
1687                         goto e_ctx;
1688         }
1689
1690         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1691                              CCP_PASSTHRU_BYTESWAP_256BIT);
1692         if (ret) {
1693                 cmd->engine_error = cmd_q->cmd_error;
1694                 goto e_ctx;
1695         }
1696
1697         if (sha->src) {
1698                 /* Send data to the CCP SHA engine; block_size is set above */
1699                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1700                                     block_size, DMA_TO_DEVICE);
1701                 if (ret)
1702                         goto e_ctx;
1703
1704                 while (src.sg_wa.bytes_left) {
1705                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1706                         if (sha->final && !src.sg_wa.bytes_left)
1707                                 op.eom = 1;
1708
1709                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1710                         if (ret) {
1711                                 cmd->engine_error = cmd_q->cmd_error;
1712                                 goto e_data;
1713                         }
1714
1715                         ccp_process_data(&src, NULL, &op);
1716                 }
1717         } else {
1718                 op.eom = 1;
1719                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1720                 if (ret) {
1721                         cmd->engine_error = cmd_q->cmd_error;
1722                         goto e_data;
1723                 }
1724         }
1725
1726         /* Retrieve the SHA context - convert from LE to BE using
1727          * 32-byte (256-bit) byteswapping to BE
1728          */
1729         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1730                                CCP_PASSTHRU_BYTESWAP_256BIT);
1731         if (ret) {
1732                 cmd->engine_error = cmd_q->cmd_error;
1733                 goto e_data;
1734         }
1735
1736         if (sha->final) {
1737                 /* Finishing up, so get the digest */
1738                 switch (sha->type) {
1739                 case CCP_SHA_TYPE_1:
1740                 case CCP_SHA_TYPE_224:
1741                 case CCP_SHA_TYPE_256:
1742                         ccp_get_dm_area(&ctx, ooffset,
1743                                         sha->ctx, 0,
1744                                         digest_size);
1745                         break;
1746                 case CCP_SHA_TYPE_384:
1747                 case CCP_SHA_TYPE_512:
1748                         ccp_get_dm_area(&ctx, 0,
1749                                         sha->ctx, LSB_ITEM_SIZE - ooffset,
1750                                         LSB_ITEM_SIZE);
1751                         ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1752                                         sha->ctx, 0,
1753                                         LSB_ITEM_SIZE - ooffset);
1754                         break;
1755                 default:
1756                         ret = -EINVAL;
1757                         goto e_data;
1758                 }
1759         } else {
1760                 /* Stash the context */
1761                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1762                                 sb_count * CCP_SB_BYTES);
1763         }
1764
1765         if (sha->final && sha->opad) {
1766                 /* HMAC operation, recursively perform final SHA */
1767                 struct ccp_cmd hmac_cmd;
1768                 struct scatterlist sg;
1769                 u8 *hmac_buf;
1770
1771                 if (sha->opad_len != block_size) {
1772                         ret = -EINVAL;
1773                         goto e_data;
1774                 }
1775
1776                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1777                 if (!hmac_buf) {
1778                         ret = -ENOMEM;
1779                         goto e_data;
1780                 }
1781                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1782
1783                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1784                 switch (sha->type) {
1785                 case CCP_SHA_TYPE_1:
1786                 case CCP_SHA_TYPE_224:
1787                 case CCP_SHA_TYPE_256:
1788                         memcpy(hmac_buf + block_size,
1789                                ctx.address + ooffset,
1790                                digest_size);
1791                         break;
1792                 case CCP_SHA_TYPE_384:
1793                 case CCP_SHA_TYPE_512:
1794                         memcpy(hmac_buf + block_size,
1795                                ctx.address + LSB_ITEM_SIZE + ooffset,
1796                                LSB_ITEM_SIZE);
1797                         memcpy(hmac_buf + block_size +
1798                                (LSB_ITEM_SIZE - ooffset),
1799                                ctx.address,
1800                                LSB_ITEM_SIZE);
1801                         break;
1802                 default:
1803                         kfree(hmac_buf);
1804                         ret = -EINVAL;
1805                         goto e_data;
1806                 }
1807
1808                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1809                 hmac_cmd.engine = CCP_ENGINE_SHA;
1810                 hmac_cmd.u.sha.type = sha->type;
1811                 hmac_cmd.u.sha.ctx = sha->ctx;
1812                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1813                 hmac_cmd.u.sha.src = &sg;
1814                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1815                 hmac_cmd.u.sha.opad = NULL;
1816                 hmac_cmd.u.sha.opad_len = 0;
1817                 hmac_cmd.u.sha.first = 1;
1818                 hmac_cmd.u.sha.final = 1;
1819                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1820
1821                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1822                 if (ret)
1823                         cmd->engine_error = hmac_cmd.engine_error;
1824
1825                 kfree(hmac_buf);
1826         }
1827
1828 e_data:
1829         if (sha->src)
1830                 ccp_free_data(&src, cmd_q);
1831
1832 e_ctx:
1833         ccp_dm_free(&ctx);
1834
1835         return ret;
1836 }
1837
1838 static noinline_for_stack int
1839 ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1840 {
1841         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1842         struct ccp_dm_workarea exp, src, dst;
1843         struct ccp_op op;
1844         unsigned int sb_count, i_len, o_len;
1845         int ret;
1846
1847         /* Check against the maximum allowable size, in bits */
1848         if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1849                 return -EINVAL;
1850
1851         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1852                 return -EINVAL;
1853
1854         memset(&op, 0, sizeof(op));
1855         op.cmd_q = cmd_q;
1856         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1857
1858         /* The RSA modulus must precede the message being acted upon, so
1859          * it must be copied to a DMA area where the message and the
1860          * modulus can be concatenated.  Therefore the input buffer
1861          * length required is twice the output buffer length (which
1862          * must be a multiple of 256-bits).  Compute o_len, i_len in bytes.
1863          * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1864          * required.
1865          */
1866         o_len = 32 * ((rsa->key_size + 255) / 256);
1867         i_len = o_len * 2;
1868
1869         sb_count = 0;
1870         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1871                 /* sb_count is the number of storage block slots required
1872                  * for the modulus.
1873                  */
1874                 sb_count = o_len / CCP_SB_BYTES;
1875                 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1876                                                                 sb_count);
1877                 if (!op.sb_key)
1878                         return -EIO;
1879         } else {
1880                 /* A version 5 device allows a modulus size that will not fit
1881                  * in the LSB, so the command will transfer it from memory.
1882                  * Set the sb key to the default, even though it's not used.
1883                  */
1884                 op.sb_key = cmd_q->sb_key;
1885         }
1886
1887         /* The RSA exponent must be in little endian format. Reverse its
1888          * byte order.
1889          */
1890         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1891         if (ret)
1892                 goto e_sb;
1893
1894         ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1895         if (ret)
1896                 goto e_exp;
1897
1898         if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1899                 /* Copy the exponent to the local storage block, using
1900                  * as many 32-byte blocks as were allocated above. It's
1901                  * already little endian, so no further change is required.
1902                  */
1903                 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1904                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1905                 if (ret) {
1906                         cmd->engine_error = cmd_q->cmd_error;
1907                         goto e_exp;
1908                 }
1909         } else {
1910                 /* The exponent can be retrieved from memory via DMA. */
1911                 op.exp.u.dma.address = exp.dma.address;
1912                 op.exp.u.dma.offset = 0;
1913         }
1914
1915         /* Concatenate the modulus and the message. Both the modulus and
1916          * the operands must be in little endian format.  Since the input
1917          * is in big endian format it must be converted.
1918          */
1919         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1920         if (ret)
1921                 goto e_exp;
1922
1923         ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1924         if (ret)
1925                 goto e_src;
1926         ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1927         if (ret)
1928                 goto e_src;
1929
1930         /* Prepare the output area for the operation */
1931         ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1932         if (ret)
1933                 goto e_src;
1934
1935         op.soc = 1;
1936         op.src.u.dma.address = src.dma.address;
1937         op.src.u.dma.offset = 0;
1938         op.src.u.dma.length = i_len;
1939         op.dst.u.dma.address = dst.dma.address;
1940         op.dst.u.dma.offset = 0;
1941         op.dst.u.dma.length = o_len;
1942
1943         op.u.rsa.mod_size = rsa->key_size;
1944         op.u.rsa.input_len = i_len;
1945
1946         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1947         if (ret) {
1948                 cmd->engine_error = cmd_q->cmd_error;
1949                 goto e_dst;
1950         }
1951
1952         ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1953
1954 e_dst:
1955         ccp_dm_free(&dst);
1956
1957 e_src:
1958         ccp_dm_free(&src);
1959
1960 e_exp:
1961         ccp_dm_free(&exp);
1962
1963 e_sb:
1964         if (sb_count)
1965                 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1966
1967         return ret;
1968 }
1969
1970 static noinline_for_stack int
1971 ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1972 {
1973         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1974         struct ccp_dm_workarea mask;
1975         struct ccp_data src, dst;
1976         struct ccp_op op;
1977         bool in_place = false;
1978         unsigned int i;
1979         int ret = 0;
1980
1981         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1982                 return -EINVAL;
1983
1984         if (!pt->src || !pt->dst)
1985                 return -EINVAL;
1986
1987         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1988                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1989                         return -EINVAL;
1990                 if (!pt->mask)
1991                         return -EINVAL;
1992         }
1993
1994         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1995
1996         memset(&op, 0, sizeof(op));
1997         op.cmd_q = cmd_q;
1998         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1999
2000         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2001                 /* Load the mask */
2002                 op.sb_key = cmd_q->sb_key;
2003
2004                 ret = ccp_init_dm_workarea(&mask, cmd_q,
2005                                            CCP_PASSTHRU_SB_COUNT *
2006                                            CCP_SB_BYTES,
2007                                            DMA_TO_DEVICE);
2008                 if (ret)
2009                         return ret;
2010
2011                 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
2012                 if (ret)
2013                         goto e_mask;
2014                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2015                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2016                 if (ret) {
2017                         cmd->engine_error = cmd_q->cmd_error;
2018                         goto e_mask;
2019                 }
2020         }
2021
2022         /* Prepare the input and output data workareas. For in-place
2023          * operations we need to set the dma direction to BIDIRECTIONAL
2024          * and copy the src workarea to the dst workarea.
2025          */
2026         if (sg_virt(pt->src) == sg_virt(pt->dst))
2027                 in_place = true;
2028
2029         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2030                             CCP_PASSTHRU_MASKSIZE,
2031                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2032         if (ret)
2033                 goto e_mask;
2034
2035         if (in_place) {
2036                 dst = src;
2037         } else {
2038                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2039                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2040                 if (ret)
2041                         goto e_src;
2042         }
2043
2044         /* Send data to the CCP Passthru engine
2045          *   Because the CCP engine works on a single source and destination
2046          *   dma address at a time, each entry in the source scatterlist
2047          *   (after the dma_map_sg call) must be less than or equal to the
2048          *   (remaining) length in the destination scatterlist entry and the
2049          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2050          */
2051         dst.sg_wa.sg_used = 0;
2052         for (i = 1; i <= src.sg_wa.dma_count; i++) {
2053                 if (!dst.sg_wa.sg ||
2054                     (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2055                         ret = -EINVAL;
2056                         goto e_dst;
2057                 }
2058
2059                 if (i == src.sg_wa.dma_count) {
2060                         op.eom = 1;
2061                         op.soc = 1;
2062                 }
2063
2064                 op.src.type = CCP_MEMTYPE_SYSTEM;
2065                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2066                 op.src.u.dma.offset = 0;
2067                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2068
2069                 op.dst.type = CCP_MEMTYPE_SYSTEM;
2070                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2071                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2072                 op.dst.u.dma.length = op.src.u.dma.length;
2073
2074                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2075                 if (ret) {
2076                         cmd->engine_error = cmd_q->cmd_error;
2077                         goto e_dst;
2078                 }
2079
2080                 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2081                 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2082                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2083                         dst.sg_wa.sg_used = 0;
2084                 }
2085                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2086         }
2087
2088 e_dst:
2089         if (!in_place)
2090                 ccp_free_data(&dst, cmd_q);
2091
2092 e_src:
2093         ccp_free_data(&src, cmd_q);
2094
2095 e_mask:
2096         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2097                 ccp_dm_free(&mask);
2098
2099         return ret;
2100 }
2101
2102 static noinline_for_stack int
2103 ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2104                                       struct ccp_cmd *cmd)
2105 {
2106         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2107         struct ccp_dm_workarea mask;
2108         struct ccp_op op;
2109         int ret;
2110
2111         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2112                 return -EINVAL;
2113
2114         if (!pt->src_dma || !pt->dst_dma)
2115                 return -EINVAL;
2116
2117         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2118                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2119                         return -EINVAL;
2120                 if (!pt->mask)
2121                         return -EINVAL;
2122         }
2123
2124         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2125
2126         memset(&op, 0, sizeof(op));
2127         op.cmd_q = cmd_q;
2128         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2129
2130         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2131                 /* Load the mask */
2132                 op.sb_key = cmd_q->sb_key;
2133
2134                 mask.length = pt->mask_len;
2135                 mask.dma.address = pt->mask;
2136                 mask.dma.length = pt->mask_len;
2137
2138                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2139                                      CCP_PASSTHRU_BYTESWAP_NOOP);
2140                 if (ret) {
2141                         cmd->engine_error = cmd_q->cmd_error;
2142                         return ret;
2143                 }
2144         }
2145
2146         /* Send data to the CCP Passthru engine */
2147         op.eom = 1;
2148         op.soc = 1;
2149
2150         op.src.type = CCP_MEMTYPE_SYSTEM;
2151         op.src.u.dma.address = pt->src_dma;
2152         op.src.u.dma.offset = 0;
2153         op.src.u.dma.length = pt->src_len;
2154
2155         op.dst.type = CCP_MEMTYPE_SYSTEM;
2156         op.dst.u.dma.address = pt->dst_dma;
2157         op.dst.u.dma.offset = 0;
2158         op.dst.u.dma.length = pt->src_len;
2159
2160         ret = cmd_q->ccp->vdata->perform->passthru(&op);
2161         if (ret)
2162                 cmd->engine_error = cmd_q->cmd_error;
2163
2164         return ret;
2165 }
2166
2167 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2168 {
2169         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2170         struct ccp_dm_workarea src, dst;
2171         struct ccp_op op;
2172         int ret;
2173         u8 *save;
2174
2175         if (!ecc->u.mm.operand_1 ||
2176             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2177                 return -EINVAL;
2178
2179         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2180                 if (!ecc->u.mm.operand_2 ||
2181                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2182                         return -EINVAL;
2183
2184         if (!ecc->u.mm.result ||
2185             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2186                 return -EINVAL;
2187
2188         memset(&op, 0, sizeof(op));
2189         op.cmd_q = cmd_q;
2190         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2191
2192         /* Concatenate the modulus and the operands. Both the modulus and
2193          * the operands must be in little endian format.  Since the input
2194          * is in big endian format it must be converted and placed in a
2195          * fixed length buffer.
2196          */
2197         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2198                                    DMA_TO_DEVICE);
2199         if (ret)
2200                 return ret;
2201
2202         /* Save the workarea address since it is updated in order to perform
2203          * the concatenation
2204          */
2205         save = src.address;
2206
2207         /* Copy the ECC modulus */
2208         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2209         if (ret)
2210                 goto e_src;
2211         src.address += CCP_ECC_OPERAND_SIZE;
2212
2213         /* Copy the first operand */
2214         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2215                                       ecc->u.mm.operand_1_len);
2216         if (ret)
2217                 goto e_src;
2218         src.address += CCP_ECC_OPERAND_SIZE;
2219
2220         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2221                 /* Copy the second operand */
2222                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2223                                               ecc->u.mm.operand_2_len);
2224                 if (ret)
2225                         goto e_src;
2226                 src.address += CCP_ECC_OPERAND_SIZE;
2227         }
2228
2229         /* Restore the workarea address */
2230         src.address = save;
2231
2232         /* Prepare the output area for the operation */
2233         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2234                                    DMA_FROM_DEVICE);
2235         if (ret)
2236                 goto e_src;
2237
2238         op.soc = 1;
2239         op.src.u.dma.address = src.dma.address;
2240         op.src.u.dma.offset = 0;
2241         op.src.u.dma.length = src.length;
2242         op.dst.u.dma.address = dst.dma.address;
2243         op.dst.u.dma.offset = 0;
2244         op.dst.u.dma.length = dst.length;
2245
2246         op.u.ecc.function = cmd->u.ecc.function;
2247
2248         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2249         if (ret) {
2250                 cmd->engine_error = cmd_q->cmd_error;
2251                 goto e_dst;
2252         }
2253
2254         ecc->ecc_result = le16_to_cpup(
2255                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2256         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2257                 ret = -EIO;
2258                 goto e_dst;
2259         }
2260
2261         /* Save the ECC result */
2262         ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2263                                 CCP_ECC_MODULUS_BYTES);
2264
2265 e_dst:
2266         ccp_dm_free(&dst);
2267
2268 e_src:
2269         ccp_dm_free(&src);
2270
2271         return ret;
2272 }
2273
2274 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2275 {
2276         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2277         struct ccp_dm_workarea src, dst;
2278         struct ccp_op op;
2279         int ret;
2280         u8 *save;
2281
2282         if (!ecc->u.pm.point_1.x ||
2283             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2284             !ecc->u.pm.point_1.y ||
2285             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2286                 return -EINVAL;
2287
2288         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2289                 if (!ecc->u.pm.point_2.x ||
2290                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2291                     !ecc->u.pm.point_2.y ||
2292                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2293                         return -EINVAL;
2294         } else {
2295                 if (!ecc->u.pm.domain_a ||
2296                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2297                         return -EINVAL;
2298
2299                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2300                         if (!ecc->u.pm.scalar ||
2301                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2302                                 return -EINVAL;
2303         }
2304
2305         if (!ecc->u.pm.result.x ||
2306             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2307             !ecc->u.pm.result.y ||
2308             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2309                 return -EINVAL;
2310
2311         memset(&op, 0, sizeof(op));
2312         op.cmd_q = cmd_q;
2313         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2314
2315         /* Concatenate the modulus and the operands. Both the modulus and
2316          * the operands must be in little endian format.  Since the input
2317          * is in big endian format it must be converted and placed in a
2318          * fixed length buffer.
2319          */
2320         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2321                                    DMA_TO_DEVICE);
2322         if (ret)
2323                 return ret;
2324
2325         /* Save the workarea address since it is updated in order to perform
2326          * the concatenation
2327          */
2328         save = src.address;
2329
2330         /* Copy the ECC modulus */
2331         ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2332         if (ret)
2333                 goto e_src;
2334         src.address += CCP_ECC_OPERAND_SIZE;
2335
2336         /* Copy the first point X and Y coordinate */
2337         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2338                                       ecc->u.pm.point_1.x_len);
2339         if (ret)
2340                 goto e_src;
2341         src.address += CCP_ECC_OPERAND_SIZE;
2342         ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2343                                       ecc->u.pm.point_1.y_len);
2344         if (ret)
2345                 goto e_src;
2346         src.address += CCP_ECC_OPERAND_SIZE;
2347
2348         /* Set the first point Z coordinate to 1 */
2349         *src.address = 0x01;
2350         src.address += CCP_ECC_OPERAND_SIZE;
2351
2352         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2353                 /* Copy the second point X and Y coordinate */
2354                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2355                                               ecc->u.pm.point_2.x_len);
2356                 if (ret)
2357                         goto e_src;
2358                 src.address += CCP_ECC_OPERAND_SIZE;
2359                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2360                                               ecc->u.pm.point_2.y_len);
2361                 if (ret)
2362                         goto e_src;
2363                 src.address += CCP_ECC_OPERAND_SIZE;
2364
2365                 /* Set the second point Z coordinate to 1 */
2366                 *src.address = 0x01;
2367                 src.address += CCP_ECC_OPERAND_SIZE;
2368         } else {
2369                 /* Copy the Domain "a" parameter */
2370                 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2371                                               ecc->u.pm.domain_a_len);
2372                 if (ret)
2373                         goto e_src;
2374                 src.address += CCP_ECC_OPERAND_SIZE;
2375
2376                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2377                         /* Copy the scalar value */
2378                         ret = ccp_reverse_set_dm_area(&src, 0,
2379                                                       ecc->u.pm.scalar, 0,
2380                                                       ecc->u.pm.scalar_len);
2381                         if (ret)
2382                                 goto e_src;
2383                         src.address += CCP_ECC_OPERAND_SIZE;
2384                 }
2385         }
2386
2387         /* Restore the workarea address */
2388         src.address = save;
2389
2390         /* Prepare the output area for the operation */
2391         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2392                                    DMA_FROM_DEVICE);
2393         if (ret)
2394                 goto e_src;
2395
2396         op.soc = 1;
2397         op.src.u.dma.address = src.dma.address;
2398         op.src.u.dma.offset = 0;
2399         op.src.u.dma.length = src.length;
2400         op.dst.u.dma.address = dst.dma.address;
2401         op.dst.u.dma.offset = 0;
2402         op.dst.u.dma.length = dst.length;
2403
2404         op.u.ecc.function = cmd->u.ecc.function;
2405
2406         ret = cmd_q->ccp->vdata->perform->ecc(&op);
2407         if (ret) {
2408                 cmd->engine_error = cmd_q->cmd_error;
2409                 goto e_dst;
2410         }
2411
2412         ecc->ecc_result = le16_to_cpup(
2413                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2414         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2415                 ret = -EIO;
2416                 goto e_dst;
2417         }
2418
2419         /* Save the workarea address since it is updated as we walk through
2420          * to copy the point math result
2421          */
2422         save = dst.address;
2423
2424         /* Save the ECC result X and Y coordinates */
2425         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2426                                 CCP_ECC_MODULUS_BYTES);
2427         dst.address += CCP_ECC_OUTPUT_SIZE;
2428         ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2429                                 CCP_ECC_MODULUS_BYTES);
2430         dst.address += CCP_ECC_OUTPUT_SIZE;
2431
2432         /* Restore the workarea address */
2433         dst.address = save;
2434
2435 e_dst:
2436         ccp_dm_free(&dst);
2437
2438 e_src:
2439         ccp_dm_free(&src);
2440
2441         return ret;
2442 }
2443
2444 static noinline_for_stack int
2445 ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2446 {
2447         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2448
2449         ecc->ecc_result = 0;
2450
2451         if (!ecc->mod ||
2452             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2453                 return -EINVAL;
2454
2455         switch (ecc->function) {
2456         case CCP_ECC_FUNCTION_MMUL_384BIT:
2457         case CCP_ECC_FUNCTION_MADD_384BIT:
2458         case CCP_ECC_FUNCTION_MINV_384BIT:
2459                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2460
2461         case CCP_ECC_FUNCTION_PADD_384BIT:
2462         case CCP_ECC_FUNCTION_PMUL_384BIT:
2463         case CCP_ECC_FUNCTION_PDBL_384BIT:
2464                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2465
2466         default:
2467                 return -EINVAL;
2468         }
2469 }
2470
2471 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2472 {
2473         int ret;
2474
2475         cmd->engine_error = 0;
2476         cmd_q->cmd_error = 0;
2477         cmd_q->int_rcvd = 0;
2478         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2479
2480         switch (cmd->engine) {
2481         case CCP_ENGINE_AES:
2482                 switch (cmd->u.aes.mode) {
2483                 case CCP_AES_MODE_CMAC:
2484                         ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2485                         break;
2486                 case CCP_AES_MODE_GCM:
2487                         ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2488                         break;
2489                 default:
2490                         ret = ccp_run_aes_cmd(cmd_q, cmd);
2491                         break;
2492                 }
2493                 break;
2494         case CCP_ENGINE_XTS_AES_128:
2495                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2496                 break;
2497         case CCP_ENGINE_DES3:
2498                 ret = ccp_run_des3_cmd(cmd_q, cmd);
2499                 break;
2500         case CCP_ENGINE_SHA:
2501                 ret = ccp_run_sha_cmd(cmd_q, cmd);
2502                 break;
2503         case CCP_ENGINE_RSA:
2504                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2505                 break;
2506         case CCP_ENGINE_PASSTHRU:
2507                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2508                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2509                 else
2510                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
2511                 break;
2512         case CCP_ENGINE_ECC:
2513                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2514                 break;
2515         default:
2516                 ret = -EINVAL;
2517         }
2518
2519         return ret;
2520 }