drivers/crypto/ccp/ccp-ops.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  * Author: Gary R Hook <gary.hook@amd.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/module.h>
  15 #include <linux/kernel.h>
  16 #include <linux/pci.h>
  17 #include <linux/interrupt.h>
  18 #include <crypto/scatterwalk.h>
  19 #include <linux/ccp.h>
  20
  21 #include "ccp-dev.h"
  22
  23 /* SHA initial context values */
  24 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
  25         cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
  26         cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
  27         cpu_to_be32(SHA1_H4),
  28 };
  29
  30 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  31         cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
  32         cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
  33         cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
  34         cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
  35 };
  36
  37 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
  38         cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
  39         cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
  40         cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
  41         cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
  42 };
  43
  44 #define CCP_NEW_JOBID(ccp)      ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
  45                                         ccp_gen_jobid(ccp) : 0)
  46
  47 static u32 ccp_gen_jobid(struct ccp_device *ccp)
  48 {
  49         return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
  50 }
  51
  52 static void ccp_sg_free(struct ccp_sg_workarea *wa)
  53 {
  54         if (wa->dma_count)
  55                 dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
  56
  57         wa->dma_count = 0;
  58 }
  59
  60 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
  61                                 struct scatterlist *sg, u64 len,
  62                                 enum dma_data_direction dma_dir)
  63 {
  64         memset(wa, 0, sizeof(*wa));
  65
  66         wa->sg = sg;
  67         if (!sg)
  68                 return 0;
  69
  70         wa->nents = sg_nents_for_len(sg, len);
  71         if (wa->nents < 0)
  72                 return wa->nents;
  73
  74         wa->bytes_left = len;
  75         wa->sg_used = 0;
  76
  77         if (len == 0)
  78                 return 0;
  79
  80         if (dma_dir == DMA_NONE)
  81                 return 0;
  82
  83         wa->dma_sg = sg;
  84         wa->dma_sg_head = sg;
  85         wa->dma_dev = dev;
  86         wa->dma_dir = dma_dir;
  87         wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
  88         if (!wa->dma_count)
  89                 return -ENOMEM;
  90
  91         return 0;
  92 }
  93
  94 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
  95 {
  96         unsigned int nbytes = min_t(u64, len, wa->bytes_left);
  97         unsigned int sg_combined_len = 0;
  98
  99         if (!wa->sg)
 100                 return;
 101
 102         wa->sg_used += nbytes;
 103         wa->bytes_left -= nbytes;
 104         if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
 105                 /* Advance to the next DMA scatterlist entry */
 106                 wa->dma_sg = sg_next(wa->dma_sg);
 107
 108                 /* In the case that the DMA mapped scatterlist has entries
 109                  * that have been merged, the non-DMA mapped scatterlist
 110                  * must be advanced multiple times for each merged entry.
 111                  * This ensures that the current non-DMA mapped entry
 112                  * corresponds to the current DMA mapped entry.
 113                  */
 114                 do {
 115                         sg_combined_len += wa->sg->length;
 116                         wa->sg = sg_next(wa->sg);
 117                 } while (wa->sg_used > sg_combined_len);
 118
 119                 wa->sg_used = 0;
 120         }
 121 }
 122
 123 static void ccp_dm_free(struct ccp_dm_workarea *wa)
 124 {
 125         if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
 126                 if (wa->address)
 127                         dma_pool_free(wa->dma_pool, wa->address,
 128                                       wa->dma.address);
 129         } else {
 130                 if (wa->dma.address)
 131                         dma_unmap_single(wa->dev, wa->dma.address, wa->length,
 132                                          wa->dma.dir);
 133                 kfree(wa->address);
 134         }
 135
 136         wa->address = NULL;
 137         wa->dma.address = 0;
 138 }
 139
 140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
 141                                 struct ccp_cmd_queue *cmd_q,
 142                                 unsigned int len,
 143                                 enum dma_data_direction dir)
 144 {
 145         memset(wa, 0, sizeof(*wa));
 146
 147         if (!len)
 148                 return 0;
 149
 150         wa->dev = cmd_q->ccp->dev;
 151         wa->length = len;
 152
 153         if (len <= CCP_DMAPOOL_MAX_SIZE) {
 154                 wa->dma_pool = cmd_q->dma_pool;
 155
 156                 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
 157                                              &wa->dma.address);
 158                 if (!wa->address)
 159                         return -ENOMEM;
 160
 161                 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
 162
 163                 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
 164         } else {
 165                 wa->address = kzalloc(len, GFP_KERNEL);
 166                 if (!wa->address)
 167                         return -ENOMEM;
 168
 169                 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
 170                                                  dir);
 171                 if (!wa->dma.address)
 172                         return -ENOMEM;
 173
 174                 wa->dma.length = len;
 175         }
 176         wa->dma.dir = dir;
 177
 178         return 0;
 179 }
 180
 181 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 182                             struct scatterlist *sg, unsigned int sg_offset,
 183                             unsigned int len)
 184 {
 185         WARN_ON(!wa->address);
 186
 187         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 188                                  0);
 189 }
 190
 191 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
 192                             struct scatterlist *sg, unsigned int sg_offset,
 193                             unsigned int len)
 194 {
 195         WARN_ON(!wa->address);
 196
 197         scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
 198                                  1);
 199 }
 200
 201 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
 202                                    struct scatterlist *sg,
 203                                    unsigned int len, unsigned int se_len,
 204                                    bool sign_extend)
 205 {
 206         unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
 207         u8 buffer[CCP_REVERSE_BUF_SIZE];
 208
 209         if (WARN_ON(se_len > sizeof(buffer)))
 210                 return -EINVAL;
 211
 212         sg_offset = len;
 213         dm_offset = 0;
 214         nbytes = len;
 215         while (nbytes) {
 216                 sb_len = min_t(unsigned int, nbytes, se_len);
 217                 sg_offset -= sb_len;
 218
 219                 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 0);
 220                 for (i = 0; i < sb_len; i++)
 221                         wa->address[dm_offset + i] = buffer[sb_len - i - 1];
 222
 223                 dm_offset += sb_len;
 224                 nbytes -= sb_len;
 225
 226                 if ((sb_len != se_len) && sign_extend) {
 227                         /* Must sign-extend to nearest sign-extend length */
 228                         if (wa->address[dm_offset - 1] & 0x80)
 229                                 memset(wa->address + dm_offset, 0xff,
 230                                        se_len - sb_len);
 231                 }
 232         }
 233
 234         return 0;
 235 }
 236
 237 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
 238                                     struct scatterlist *sg,
 239                                     unsigned int len)
 240 {
 241         unsigned int nbytes, sg_offset, dm_offset, sb_len, i;
 242         u8 buffer[CCP_REVERSE_BUF_SIZE];
 243
 244         sg_offset = 0;
 245         dm_offset = len;
 246         nbytes = len;
 247         while (nbytes) {
 248                 sb_len = min_t(unsigned int, nbytes, sizeof(buffer));
 249                 dm_offset -= sb_len;
 250
 251                 for (i = 0; i < sb_len; i++)
 252                         buffer[sb_len - i - 1] = wa->address[dm_offset + i];
 253                 scatterwalk_map_and_copy(buffer, sg, sg_offset, sb_len, 1);
 254
 255                 sg_offset += sb_len;
 256                 nbytes -= sb_len;
 257         }
 258 }
 259
 260 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
 261 {
 262         ccp_dm_free(&data->dm_wa);
 263         ccp_sg_free(&data->sg_wa);
 264 }
 265
 266 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
 267                          struct scatterlist *sg, u64 sg_len,
 268                          unsigned int dm_len,
 269                          enum dma_data_direction dir)
 270 {
 271         int ret;
 272
 273         memset(data, 0, sizeof(*data));
 274
 275         ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
 276                                    dir);
 277         if (ret)
 278                 goto e_err;
 279
 280         ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
 281         if (ret)
 282                 goto e_err;
 283
 284         return 0;
 285
 286 e_err:
 287         ccp_free_data(data, cmd_q);
 288
 289         return ret;
 290 }
 291
 292 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
 293 {
 294         struct ccp_sg_workarea *sg_wa = &data->sg_wa;
 295         struct ccp_dm_workarea *dm_wa = &data->dm_wa;
 296         unsigned int buf_count, nbytes;
 297
 298         /* Clear the buffer if setting it */
 299         if (!from)
 300                 memset(dm_wa->address, 0, dm_wa->length);
 301
 302         if (!sg_wa->sg)
 303                 return 0;
 304
 305         /* Perform the copy operation
 306          *   nbytes will always be <= UINT_MAX because dm_wa->length is
 307          *   an unsigned int
 308          */
 309         nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
 310         scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
 311                                  nbytes, from);
 312
 313         /* Update the structures and generate the count */
 314         buf_count = 0;
 315         while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
 316                 nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
 317                              dm_wa->length - buf_count);
 318                 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
 319
 320                 buf_count += nbytes;
 321                 ccp_update_sg_workarea(sg_wa, nbytes);
 322         }
 323
 324         return buf_count;
 325 }
 326
 327 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
 328 {
 329         return ccp_queue_buf(data, 0);
 330 }
 331
 332 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
 333 {
 334         return ccp_queue_buf(data, 1);
 335 }
 336
 337 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
 338                              struct ccp_op *op, unsigned int block_size,
 339                              bool blocksize_op)
 340 {
 341         unsigned int sg_src_len, sg_dst_len, op_len;
 342
 343         /* The CCP can only DMA from/to one address each per operation. This
 344          * requires that we find the smallest DMA area between the source
 345          * and destination. The resulting len values will always be <= UINT_MAX
 346          * because the dma length is an unsigned int.
 347          */
 348         sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
 349         sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
 350
 351         if (dst) {
 352                 sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
 353                 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
 354                 op_len = min(sg_src_len, sg_dst_len);
 355         } else {
 356                 op_len = sg_src_len;
 357         }
 358
 359         /* The data operation length will be at least block_size in length
 360          * or the smaller of available sg room remaining for the source or
 361          * the destination
 362          */
 363         op_len = max(op_len, block_size);
 364
 365         /* Unless we have to buffer data, there's no reason to wait */
 366         op->soc = 0;
 367
 368         if (sg_src_len < block_size) {
 369                 /* Not enough data in the sg element, so it
 370                  * needs to be buffered into a blocksize chunk
 371                  */
 372                 int cp_len = ccp_fill_queue_buf(src);
 373
 374                 op->soc = 1;
 375                 op->src.u.dma.address = src->dm_wa.dma.address;
 376                 op->src.u.dma.offset = 0;
 377                 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
 378         } else {
 379                 /* Enough data in the sg element, but we need to
 380                  * adjust for any previously copied data
 381                  */
 382                 op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
 383                 op->src.u.dma.offset = src->sg_wa.sg_used;
 384                 op->src.u.dma.length = op_len & ~(block_size - 1);
 385
 386                 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
 387         }
 388
 389         if (dst) {
 390                 if (sg_dst_len < block_size) {
 391                         /* Not enough room in the sg element or we're on the
 392                          * last piece of data (when using padding), so the
 393                          * output needs to be buffered into a blocksize chunk
 394                          */
 395                         op->soc = 1;
 396                         op->dst.u.dma.address = dst->dm_wa.dma.address;
 397                         op->dst.u.dma.offset = 0;
 398                         op->dst.u.dma.length = op->src.u.dma.length;
 399                 } else {
 400                         /* Enough room in the sg element, but we need to
 401                          * adjust for any previously used area
 402                          */
 403                         op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
 404                         op->dst.u.dma.offset = dst->sg_wa.sg_used;
 405                         op->dst.u.dma.length = op->src.u.dma.length;
 406                 }
 407         }
 408 }
 409
 410 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
 411                              struct ccp_op *op)
 412 {
 413         op->init = 0;
 414
 415         if (dst) {
 416                 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
 417                         ccp_empty_queue_buf(dst);
 418                 else
 419                         ccp_update_sg_workarea(&dst->sg_wa,
 420                                                op->dst.u.dma.length);
 421         }
 422 }
 423
 424 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
 425                                struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 426                                u32 byte_swap, bool from)
 427 {
 428         struct ccp_op op;
 429
 430         memset(&op, 0, sizeof(op));
 431
 432         op.cmd_q = cmd_q;
 433         op.jobid = jobid;
 434         op.eom = 1;
 435
 436         if (from) {
 437                 op.soc = 1;
 438                 op.src.type = CCP_MEMTYPE_SB;
 439                 op.src.u.sb = sb;
 440                 op.dst.type = CCP_MEMTYPE_SYSTEM;
 441                 op.dst.u.dma.address = wa->dma.address;
 442                 op.dst.u.dma.length = wa->length;
 443         } else {
 444                 op.src.type = CCP_MEMTYPE_SYSTEM;
 445                 op.src.u.dma.address = wa->dma.address;
 446                 op.src.u.dma.length = wa->length;
 447                 op.dst.type = CCP_MEMTYPE_SB;
 448                 op.dst.u.sb = sb;
 449         }
 450
 451         op.u.passthru.byte_swap = byte_swap;
 452
 453         return cmd_q->ccp->vdata->perform->passthru(&op);
 454 }
 455
 456 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
 457                           struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 458                           u32 byte_swap)
 459 {
 460         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
 461 }
 462
 463 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
 464                             struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
 465                             u32 byte_swap)
 466 {
 467         return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
 468 }
 469
 470 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
 471                                 struct ccp_cmd *cmd)
 472 {
 473         struct ccp_aes_engine *aes = &cmd->u.aes;
 474         struct ccp_dm_workarea key, ctx;
 475         struct ccp_data src;
 476         struct ccp_op op;
 477         unsigned int dm_offset;
 478         int ret;
 479
 480         if (!((aes->key_len == AES_KEYSIZE_128) ||
 481               (aes->key_len == AES_KEYSIZE_192) ||
 482               (aes->key_len == AES_KEYSIZE_256)))
 483                 return -EINVAL;
 484
 485         if (aes->src_len & (AES_BLOCK_SIZE - 1))
 486                 return -EINVAL;
 487
 488         if (aes->iv_len != AES_BLOCK_SIZE)
 489                 return -EINVAL;
 490
 491         if (!aes->key || !aes->iv || !aes->src)
 492                 return -EINVAL;
 493
 494         if (aes->cmac_final) {
 495                 if (aes->cmac_key_len != AES_BLOCK_SIZE)
 496                         return -EINVAL;
 497
 498                 if (!aes->cmac_key)
 499                         return -EINVAL;
 500         }
 501
 502         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 503         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 504
 505         ret = -EIO;
 506         memset(&op, 0, sizeof(op));
 507         op.cmd_q = cmd_q;
 508         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 509         op.sb_key = cmd_q->sb_key;
 510         op.sb_ctx = cmd_q->sb_ctx;
 511         op.init = 1;
 512         op.u.aes.type = aes->type;
 513         op.u.aes.mode = aes->mode;
 514         op.u.aes.action = aes->action;
 515
 516         /* All supported key sizes fit in a single (32-byte) SB entry
 517          * and must be in little endian format. Use the 256-bit byte
 518          * swap passthru option to convert from big endian to little
 519          * endian.
 520          */
 521         ret = ccp_init_dm_workarea(&key, cmd_q,
 522                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 523                                    DMA_TO_DEVICE);
 524         if (ret)
 525                 return ret;
 526
 527         dm_offset = CCP_SB_BYTES - aes->key_len;
 528         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 529         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 530                              CCP_PASSTHRU_BYTESWAP_256BIT);
 531         if (ret) {
 532                 cmd->engine_error = cmd_q->cmd_error;
 533                 goto e_key;
 534         }
 535
 536         /* The AES context fits in a single (32-byte) SB entry and
 537          * must be in little endian format. Use the 256-bit byte swap
 538          * passthru option to convert from big endian to little endian.
 539          */
 540         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 541                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 542                                    DMA_BIDIRECTIONAL);
 543         if (ret)
 544                 goto e_key;
 545
 546         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 547         ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 548         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 549                              CCP_PASSTHRU_BYTESWAP_256BIT);
 550         if (ret) {
 551                 cmd->engine_error = cmd_q->cmd_error;
 552                 goto e_ctx;
 553         }
 554
 555         /* Send data to the CCP AES engine */
 556         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 557                             AES_BLOCK_SIZE, DMA_TO_DEVICE);
 558         if (ret)
 559                 goto e_ctx;
 560
 561         while (src.sg_wa.bytes_left) {
 562                 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
 563                 if (aes->cmac_final && !src.sg_wa.bytes_left) {
 564                         op.eom = 1;
 565
 566                         /* Push the K1/K2 key to the CCP now */
 567                         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
 568                                                op.sb_ctx,
 569                                                CCP_PASSTHRU_BYTESWAP_256BIT);
 570                         if (ret) {
 571                                 cmd->engine_error = cmd_q->cmd_error;
 572                                 goto e_src;
 573                         }
 574
 575                         ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
 576                                         aes->cmac_key_len);
 577                         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 578                                              CCP_PASSTHRU_BYTESWAP_256BIT);
 579                         if (ret) {
 580                                 cmd->engine_error = cmd_q->cmd_error;
 581                                 goto e_src;
 582                         }
 583                 }
 584
 585                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 586                 if (ret) {
 587                         cmd->engine_error = cmd_q->cmd_error;
 588                         goto e_src;
 589                 }
 590
 591                 ccp_process_data(&src, NULL, &op);
 592         }
 593
 594         /* Retrieve the AES context - convert from LE to BE using
 595          * 32-byte (256-bit) byteswapping
 596          */
 597         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 598                                CCP_PASSTHRU_BYTESWAP_256BIT);
 599         if (ret) {
 600                 cmd->engine_error = cmd_q->cmd_error;
 601                 goto e_src;
 602         }
 603
 604         /* ...but we only need AES_BLOCK_SIZE bytes */
 605         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 606         ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 607
 608 e_src:
 609         ccp_free_data(&src, cmd_q);
 610
 611 e_ctx:
 612         ccp_dm_free(&ctx);
 613
 614 e_key:
 615         ccp_dm_free(&key);
 616
 617         return ret;
 618 }
 619
 620 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 621 {
 622         struct ccp_aes_engine *aes = &cmd->u.aes;
 623         struct ccp_dm_workarea key, ctx;
 624         struct ccp_data src, dst;
 625         struct ccp_op op;
 626         unsigned int dm_offset;
 627         bool in_place = false;
 628         int ret;
 629
 630         if (aes->mode == CCP_AES_MODE_CMAC)
 631                 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
 632
 633         if (!((aes->key_len == AES_KEYSIZE_128) ||
 634               (aes->key_len == AES_KEYSIZE_192) ||
 635               (aes->key_len == AES_KEYSIZE_256)))
 636                 return -EINVAL;
 637
 638         if (((aes->mode == CCP_AES_MODE_ECB) ||
 639              (aes->mode == CCP_AES_MODE_CBC) ||
 640              (aes->mode == CCP_AES_MODE_CFB)) &&
 641             (aes->src_len & (AES_BLOCK_SIZE - 1)))
 642                 return -EINVAL;
 643
 644         if (!aes->key || !aes->src || !aes->dst)
 645                 return -EINVAL;
 646
 647         if (aes->mode != CCP_AES_MODE_ECB) {
 648                 if (aes->iv_len != AES_BLOCK_SIZE)
 649                         return -EINVAL;
 650
 651                 if (!aes->iv)
 652                         return -EINVAL;
 653         }
 654
 655         BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
 656         BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
 657
 658         ret = -EIO;
 659         memset(&op, 0, sizeof(op));
 660         op.cmd_q = cmd_q;
 661         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 662         op.sb_key = cmd_q->sb_key;
 663         op.sb_ctx = cmd_q->sb_ctx;
 664         op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
 665         op.u.aes.type = aes->type;
 666         op.u.aes.mode = aes->mode;
 667         op.u.aes.action = aes->action;
 668
 669         /* All supported key sizes fit in a single (32-byte) SB entry
 670          * and must be in little endian format. Use the 256-bit byte
 671          * swap passthru option to convert from big endian to little
 672          * endian.
 673          */
 674         ret = ccp_init_dm_workarea(&key, cmd_q,
 675                                    CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
 676                                    DMA_TO_DEVICE);
 677         if (ret)
 678                 return ret;
 679
 680         dm_offset = CCP_SB_BYTES - aes->key_len;
 681         ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
 682         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 683                              CCP_PASSTHRU_BYTESWAP_256BIT);
 684         if (ret) {
 685                 cmd->engine_error = cmd_q->cmd_error;
 686                 goto e_key;
 687         }
 688
 689         /* The AES context fits in a single (32-byte) SB entry and
 690          * must be in little endian format. Use the 256-bit byte swap
 691          * passthru option to convert from big endian to little endian.
 692          */
 693         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 694                                    CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 695                                    DMA_BIDIRECTIONAL);
 696         if (ret)
 697                 goto e_key;
 698
 699         if (aes->mode != CCP_AES_MODE_ECB) {
 700                 /* Load the AES context - convert to LE */
 701                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 702                 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 703                 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 704                                      CCP_PASSTHRU_BYTESWAP_256BIT);
 705                 if (ret) {
 706                         cmd->engine_error = cmd_q->cmd_error;
 707                         goto e_ctx;
 708                 }
 709         }
 710         switch (aes->mode) {
 711         case CCP_AES_MODE_CFB: /* CFB128 only */
 712         case CCP_AES_MODE_CTR:
 713                 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
 714                 break;
 715         default:
 716                 op.u.aes.size = 0;
 717         }
 718
 719         /* Prepare the input and output data workareas. For in-place
 720          * operations we need to set the dma direction to BIDIRECTIONAL
 721          * and copy the src workarea to the dst workarea.
 722          */
 723         if (sg_virt(aes->src) == sg_virt(aes->dst))
 724                 in_place = true;
 725
 726         ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
 727                             AES_BLOCK_SIZE,
 728                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 729         if (ret)
 730                 goto e_ctx;
 731
 732         if (in_place) {
 733                 dst = src;
 734         } else {
 735                 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
 736                                     AES_BLOCK_SIZE, DMA_FROM_DEVICE);
 737                 if (ret)
 738                         goto e_src;
 739         }
 740
 741         /* Send data to the CCP AES engine */
 742         while (src.sg_wa.bytes_left) {
 743                 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
 744                 if (!src.sg_wa.bytes_left) {
 745                         op.eom = 1;
 746
 747                         /* Since we don't retrieve the AES context in ECB
 748                          * mode we have to wait for the operation to complete
 749                          * on the last piece of data
 750                          */
 751                         if (aes->mode == CCP_AES_MODE_ECB)
 752                                 op.soc = 1;
 753                 }
 754
 755                 ret = cmd_q->ccp->vdata->perform->aes(&op);
 756                 if (ret) {
 757                         cmd->engine_error = cmd_q->cmd_error;
 758                         goto e_dst;
 759                 }
 760
 761                 ccp_process_data(&src, &dst, &op);
 762         }
 763
 764         if (aes->mode != CCP_AES_MODE_ECB) {
 765                 /* Retrieve the AES context - convert from LE to BE using
 766                  * 32-byte (256-bit) byteswapping
 767                  */
 768                 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 769                                        CCP_PASSTHRU_BYTESWAP_256BIT);
 770                 if (ret) {
 771                         cmd->engine_error = cmd_q->cmd_error;
 772                         goto e_dst;
 773                 }
 774
 775                 /* ...but we only need AES_BLOCK_SIZE bytes */
 776                 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 777                 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
 778         }
 779
 780 e_dst:
 781         if (!in_place)
 782                 ccp_free_data(&dst, cmd_q);
 783
 784 e_src:
 785         ccp_free_data(&src, cmd_q);
 786
 787 e_ctx:
 788         ccp_dm_free(&ctx);
 789
 790 e_key:
 791         ccp_dm_free(&key);
 792
 793         return ret;
 794 }
 795
 796 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
 797                                struct ccp_cmd *cmd)
 798 {
 799         struct ccp_xts_aes_engine *xts = &cmd->u.xts;
 800         struct ccp_dm_workarea key, ctx;
 801         struct ccp_data src, dst;
 802         struct ccp_op op;
 803         unsigned int unit_size, dm_offset;
 804         bool in_place = false;
 805         unsigned int sb_count;
 806         enum ccp_aes_type aestype;
 807         int ret;
 808
 809         switch (xts->unit_size) {
 810         case CCP_XTS_AES_UNIT_SIZE_16:
 811                 unit_size = 16;
 812                 break;
 813         case CCP_XTS_AES_UNIT_SIZE_512:
 814                 unit_size = 512;
 815                 break;
 816         case CCP_XTS_AES_UNIT_SIZE_1024:
 817                 unit_size = 1024;
 818                 break;
 819         case CCP_XTS_AES_UNIT_SIZE_2048:
 820                 unit_size = 2048;
 821                 break;
 822         case CCP_XTS_AES_UNIT_SIZE_4096:
 823                 unit_size = 4096;
 824                 break;
 825
 826         default:
 827                 return -EINVAL;
 828         }
 829
 830         if (xts->key_len == AES_KEYSIZE_128)
 831                 aestype = CCP_AES_TYPE_128;
 832         else
 833                 return -EINVAL;
 834
 835         if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
 836                 return -EINVAL;
 837
 838         if (xts->iv_len != AES_BLOCK_SIZE)
 839                 return -EINVAL;
 840
 841         if (!xts->key || !xts->iv || !xts->src || !xts->dst)
 842                 return -EINVAL;
 843
 844         BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
 845         BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
 846
 847         ret = -EIO;
 848         memset(&op, 0, sizeof(op));
 849         op.cmd_q = cmd_q;
 850         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
 851         op.sb_key = cmd_q->sb_key;
 852         op.sb_ctx = cmd_q->sb_ctx;
 853         op.init = 1;
 854         op.u.xts.type = aestype;
 855         op.u.xts.action = xts->action;
 856         op.u.xts.unit_size = xts->unit_size;
 857
 858         /* A version 3 device only supports 128-bit keys, which fits into a
 859          * single SB entry. A version 5 device uses a 512-bit vector, so two
 860          * SB entries.
 861          */
 862         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
 863                 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
 864         else
 865                 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
 866         ret = ccp_init_dm_workarea(&key, cmd_q,
 867                                    sb_count * CCP_SB_BYTES,
 868                                    DMA_TO_DEVICE);
 869         if (ret)
 870                 return ret;
 871
 872         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
 873                 /* All supported key sizes must be in little endian format.
 874                  * Use the 256-bit byte swap passthru option to convert from
 875                  * big endian to little endian.
 876                  */
 877                 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
 878                 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
 879                 ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
 880         } else {
 881                 /* Version 5 CCPs use a 512-bit space for the key: each portion
 882                  * occupies 256 bits, or one entire slot, and is zero-padded.
 883                  */
 884                 unsigned int pad;
 885
 886                 dm_offset = CCP_SB_BYTES;
 887                 pad = dm_offset - xts->key_len;
 888                 ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
 889                 ccp_set_dm_area(&key, dm_offset + pad, xts->key, xts->key_len,
 890                                 xts->key_len);
 891         }
 892         ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
 893                              CCP_PASSTHRU_BYTESWAP_256BIT);
 894         if (ret) {
 895                 cmd->engine_error = cmd_q->cmd_error;
 896                 goto e_key;
 897         }
 898
 899         /* The AES context fits in a single (32-byte) SB entry and
 900          * for XTS is already in little endian format so no byte swapping
 901          * is needed.
 902          */
 903         ret = ccp_init_dm_workarea(&ctx, cmd_q,
 904                                    CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
 905                                    DMA_BIDIRECTIONAL);
 906         if (ret)
 907                 goto e_key;
 908
 909         ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
 910         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 911                              CCP_PASSTHRU_BYTESWAP_NOOP);
 912         if (ret) {
 913                 cmd->engine_error = cmd_q->cmd_error;
 914                 goto e_ctx;
 915         }
 916
 917         /* Prepare the input and output data workareas. For in-place
 918          * operations we need to set the dma direction to BIDIRECTIONAL
 919          * and copy the src workarea to the dst workarea.
 920          */
 921         if (sg_virt(xts->src) == sg_virt(xts->dst))
 922                 in_place = true;
 923
 924         ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
 925                             unit_size,
 926                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
 927         if (ret)
 928                 goto e_ctx;
 929
 930         if (in_place) {
 931                 dst = src;
 932         } else {
 933                 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
 934                                     unit_size, DMA_FROM_DEVICE);
 935                 if (ret)
 936                         goto e_src;
 937         }
 938
 939         /* Send data to the CCP AES engine */
 940         while (src.sg_wa.bytes_left) {
 941                 ccp_prepare_data(&src, &dst, &op, unit_size, true);
 942                 if (!src.sg_wa.bytes_left)
 943                         op.eom = 1;
 944
 945                 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
 946                 if (ret) {
 947                         cmd->engine_error = cmd_q->cmd_error;
 948                         goto e_dst;
 949                 }
 950
 951                 ccp_process_data(&src, &dst, &op);
 952         }
 953
 954         /* Retrieve the AES context - convert from LE to BE using
 955          * 32-byte (256-bit) byteswapping
 956          */
 957         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
 958                                CCP_PASSTHRU_BYTESWAP_256BIT);
 959         if (ret) {
 960                 cmd->engine_error = cmd_q->cmd_error;
 961                 goto e_dst;
 962         }
 963
 964         /* ...but we only need AES_BLOCK_SIZE bytes */
 965         dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
 966         ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
 967
 968 e_dst:
 969         if (!in_place)
 970                 ccp_free_data(&dst, cmd_q);
 971
 972 e_src:
 973         ccp_free_data(&src, cmd_q);
 974
 975 e_ctx:
 976         ccp_dm_free(&ctx);
 977
 978 e_key:
 979         ccp_dm_free(&key);
 980
 981         return ret;
 982 }
 983
 984 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
 985 {
 986         struct ccp_sha_engine *sha = &cmd->u.sha;
 987         struct ccp_dm_workarea ctx;
 988         struct ccp_data src;
 989         struct ccp_op op;
 990         unsigned int ioffset, ooffset;
 991         unsigned int digest_size;
 992         int sb_count;
 993         const void *init;
 994         u64 block_size;
 995         int ctx_size;
 996         int ret;
 997
 998         switch (sha->type) {
 999         case CCP_SHA_TYPE_1:
1000                 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1001                         return -EINVAL;
1002                 block_size = SHA1_BLOCK_SIZE;
1003                 break;
1004         case CCP_SHA_TYPE_224:
1005                 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1006                         return -EINVAL;
1007                 block_size = SHA224_BLOCK_SIZE;
1008                 break;
1009         case CCP_SHA_TYPE_256:
1010                 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1011                         return -EINVAL;
1012                 block_size = SHA256_BLOCK_SIZE;
1013                 break;
1014         default:
1015                 return -EINVAL;
1016         }
1017
1018         if (!sha->ctx)
1019                 return -EINVAL;
1020
1021         if (!sha->final && (sha->src_len & (block_size - 1)))
1022                 return -EINVAL;
1023
1024         /* The version 3 device can't handle zero-length input */
1025         if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1026
1027                 if (!sha->src_len) {
1028                         unsigned int digest_len;
1029                         const u8 *sha_zero;
1030
1031                         /* Not final, just return */
1032                         if (!sha->final)
1033                                 return 0;
1034
1035                         /* CCP can't do a zero length sha operation so the
1036                          * caller must buffer the data.
1037                          */
1038                         if (sha->msg_bits)
1039                                 return -EINVAL;
1040
1041                         /* The CCP cannot perform zero-length sha operations
1042                          * so the caller is required to buffer data for the
1043                          * final operation. However, a sha operation for a
1044                          * message with a total length of zero is valid so
1045                          * known values are required to supply the result.
1046                          */
1047                         switch (sha->type) {
1048                         case CCP_SHA_TYPE_1:
1049                                 sha_zero = sha1_zero_message_hash;
1050                                 digest_len = SHA1_DIGEST_SIZE;
1051                                 break;
1052                         case CCP_SHA_TYPE_224:
1053                                 sha_zero = sha224_zero_message_hash;
1054                                 digest_len = SHA224_DIGEST_SIZE;
1055                                 break;
1056                         case CCP_SHA_TYPE_256:
1057                                 sha_zero = sha256_zero_message_hash;
1058                                 digest_len = SHA256_DIGEST_SIZE;
1059                                 break;
1060                         default:
1061                                 return -EINVAL;
1062                         }
1063
1064                         scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1065                                                  digest_len, 1);
1066
1067                         return 0;
1068                 }
1069         }
1070
1071         /* Set variables used throughout */
1072         switch (sha->type) {
1073         case CCP_SHA_TYPE_1:
1074                 digest_size = SHA1_DIGEST_SIZE;
1075                 init = (void *) ccp_sha1_init;
1076                 ctx_size = SHA1_DIGEST_SIZE;
1077                 sb_count = 1;
1078                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1079                         ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1080                 else
1081                         ooffset = ioffset = 0;
1082                 break;
1083         case CCP_SHA_TYPE_224:
1084                 digest_size = SHA224_DIGEST_SIZE;
1085                 init = (void *) ccp_sha224_init;
1086                 ctx_size = SHA256_DIGEST_SIZE;
1087                 sb_count = 1;
1088                 ioffset = 0;
1089                 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1090                         ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1091                 else
1092                         ooffset = 0;
1093                 break;
1094         case CCP_SHA_TYPE_256:
1095                 digest_size = SHA256_DIGEST_SIZE;
1096                 init = (void *) ccp_sha256_init;
1097                 ctx_size = SHA256_DIGEST_SIZE;
1098                 sb_count = 1;
1099                 ooffset = ioffset = 0;
1100                 break;
1101         default:
1102                 ret = -EINVAL;
1103                 goto e_data;
1104         }
1105
1106         /* For zero-length plaintext the src pointer is ignored;
1107          * otherwise both parts must be valid
1108          */
1109         if (sha->src_len && !sha->src)
1110                 return -EINVAL;
1111
1112         memset(&op, 0, sizeof(op));
1113         op.cmd_q = cmd_q;
1114         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1115         op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1116         op.u.sha.type = sha->type;
1117         op.u.sha.msg_bits = sha->msg_bits;
1118
1119         ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1120                                    DMA_BIDIRECTIONAL);
1121         if (ret)
1122                 return ret;
1123         if (sha->first) {
1124                 switch (sha->type) {
1125                 case CCP_SHA_TYPE_1:
1126                 case CCP_SHA_TYPE_224:
1127                 case CCP_SHA_TYPE_256:
1128                         memcpy(ctx.address + ioffset, init, ctx_size);
1129                         break;
1130                 default:
1131                         ret = -EINVAL;
1132                         goto e_ctx;
1133                 }
1134         } else {
1135                 /* Restore the context */
1136                 ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1137                                 sb_count * CCP_SB_BYTES);
1138         }
1139
1140         ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1141                              CCP_PASSTHRU_BYTESWAP_256BIT);
1142         if (ret) {
1143                 cmd->engine_error = cmd_q->cmd_error;
1144                 goto e_ctx;
1145         }
1146
1147         if (sha->src) {
1148                 /* Send data to the CCP SHA engine; block_size is set above */
1149                 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1150                                     block_size, DMA_TO_DEVICE);
1151                 if (ret)
1152                         goto e_ctx;
1153
1154                 while (src.sg_wa.bytes_left) {
1155                         ccp_prepare_data(&src, NULL, &op, block_size, false);
1156                         if (sha->final && !src.sg_wa.bytes_left)
1157                                 op.eom = 1;
1158
1159                         ret = cmd_q->ccp->vdata->perform->sha(&op);
1160                         if (ret) {
1161                                 cmd->engine_error = cmd_q->cmd_error;
1162                                 goto e_data;
1163                         }
1164
1165                         ccp_process_data(&src, NULL, &op);
1166                 }
1167         } else {
1168                 op.eom = 1;
1169                 ret = cmd_q->ccp->vdata->perform->sha(&op);
1170                 if (ret) {
1171                         cmd->engine_error = cmd_q->cmd_error;
1172                         goto e_data;
1173                 }
1174         }
1175
1176         /* Retrieve the SHA context - convert from LE to BE using
1177          * 32-byte (256-bit) byteswapping to BE
1178          */
1179         ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1180                                CCP_PASSTHRU_BYTESWAP_256BIT);
1181         if (ret) {
1182                 cmd->engine_error = cmd_q->cmd_error;
1183                 goto e_data;
1184         }
1185
1186         if (sha->final) {
1187                 /* Finishing up, so get the digest */
1188                 switch (sha->type) {
1189                 case CCP_SHA_TYPE_1:
1190                 case CCP_SHA_TYPE_224:
1191                 case CCP_SHA_TYPE_256:
1192                         ccp_get_dm_area(&ctx, ooffset,
1193                                         sha->ctx, 0,
1194                                         digest_size);
1195                         break;
1196                 default:
1197                         ret = -EINVAL;
1198                         goto e_data;
1199                 }
1200         } else {
1201                 /* Stash the context */
1202                 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1203                                 sb_count * CCP_SB_BYTES);
1204         }
1205
1206         if (sha->final && sha->opad) {
1207                 /* HMAC operation, recursively perform final SHA */
1208                 struct ccp_cmd hmac_cmd;
1209                 struct scatterlist sg;
1210                 u8 *hmac_buf;
1211
1212                 if (sha->opad_len != block_size) {
1213                         ret = -EINVAL;
1214                         goto e_data;
1215                 }
1216
1217                 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1218                 if (!hmac_buf) {
1219                         ret = -ENOMEM;
1220                         goto e_data;
1221                 }
1222                 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1223
1224                 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1225                 switch (sha->type) {
1226                 case CCP_SHA_TYPE_1:
1227                 case CCP_SHA_TYPE_224:
1228                 case CCP_SHA_TYPE_256:
1229                         memcpy(hmac_buf + block_size,
1230                                ctx.address + ooffset,
1231                                digest_size);
1232                         break;
1233                 default:
1234                         kfree(hmac_buf);
1235                         ret = -EINVAL;
1236                         goto e_data;
1237                 }
1238
1239                 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1240                 hmac_cmd.engine = CCP_ENGINE_SHA;
1241                 hmac_cmd.u.sha.type = sha->type;
1242                 hmac_cmd.u.sha.ctx = sha->ctx;
1243                 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1244                 hmac_cmd.u.sha.src = &sg;
1245                 hmac_cmd.u.sha.src_len = block_size + digest_size;
1246                 hmac_cmd.u.sha.opad = NULL;
1247                 hmac_cmd.u.sha.opad_len = 0;
1248                 hmac_cmd.u.sha.first = 1;
1249                 hmac_cmd.u.sha.final = 1;
1250                 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1251
1252                 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1253                 if (ret)
1254                         cmd->engine_error = hmac_cmd.engine_error;
1255
1256                 kfree(hmac_buf);
1257         }
1258
1259 e_data:
1260         if (sha->src)
1261                 ccp_free_data(&src, cmd_q);
1262
1263 e_ctx:
1264         ccp_dm_free(&ctx);
1265
1266         return ret;
1267 }
1268
1269 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1270 {
1271         struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1272         struct ccp_dm_workarea exp, src;
1273         struct ccp_data dst;
1274         struct ccp_op op;
1275         unsigned int sb_count, i_len, o_len;
1276         int ret;
1277
1278         if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1279                 return -EINVAL;
1280
1281         if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1282                 return -EINVAL;
1283
1284         /* The RSA modulus must precede the message being acted upon, so
1285          * it must be copied to a DMA area where the message and the
1286          * modulus can be concatenated.  Therefore the input buffer
1287          * length required is twice the output buffer length (which
1288          * must be a multiple of 256-bits).
1289          */
1290         o_len = ((rsa->key_size + 255) / 256) * 32;
1291         i_len = o_len * 2;
1292
1293         sb_count = o_len / CCP_SB_BYTES;
1294
1295         memset(&op, 0, sizeof(op));
1296         op.cmd_q = cmd_q;
1297         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1298         op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q, sb_count);
1299
1300         if (!op.sb_key)
1301                 return -EIO;
1302
1303         /* The RSA exponent may span multiple (32-byte) SB entries and must
1304          * be in little endian format. Reverse copy each 32-byte chunk
1305          * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1306          * and each byte within that chunk and do not perform any byte swap
1307          * operations on the passthru operation.
1308          */
1309         ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1310         if (ret)
1311                 goto e_sb;
1312
1313         ret = ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len,
1314                                       CCP_SB_BYTES, false);
1315         if (ret)
1316                 goto e_exp;
1317         ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1318                              CCP_PASSTHRU_BYTESWAP_NOOP);
1319         if (ret) {
1320                 cmd->engine_error = cmd_q->cmd_error;
1321                 goto e_exp;
1322         }
1323
1324         /* Concatenate the modulus and the message. Both the modulus and
1325          * the operands must be in little endian format.  Since the input
1326          * is in big endian format it must be converted.
1327          */
1328         ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1329         if (ret)
1330                 goto e_exp;
1331
1332         ret = ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len,
1333                                       CCP_SB_BYTES, false);
1334         if (ret)
1335                 goto e_src;
1336         src.address += o_len;   /* Adjust the address for the copy operation */
1337         ret = ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len,
1338                                       CCP_SB_BYTES, false);
1339         if (ret)
1340                 goto e_src;
1341         src.address -= o_len;   /* Reset the address to original value */
1342
1343         /* Prepare the output area for the operation */
1344         ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1345                             o_len, DMA_FROM_DEVICE);
1346         if (ret)
1347                 goto e_src;
1348
1349         op.soc = 1;
1350         op.src.u.dma.address = src.dma.address;
1351         op.src.u.dma.offset = 0;
1352         op.src.u.dma.length = i_len;
1353         op.dst.u.dma.address = dst.dm_wa.dma.address;
1354         op.dst.u.dma.offset = 0;
1355         op.dst.u.dma.length = o_len;
1356
1357         op.u.rsa.mod_size = rsa->key_size;
1358         op.u.rsa.input_len = i_len;
1359
1360         ret = cmd_q->ccp->vdata->perform->rsa(&op);
1361         if (ret) {
1362                 cmd->engine_error = cmd_q->cmd_error;
1363                 goto e_dst;
1364         }
1365
1366         ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1367
1368 e_dst:
1369         ccp_free_data(&dst, cmd_q);
1370
1371 e_src:
1372         ccp_dm_free(&src);
1373
1374 e_exp:
1375         ccp_dm_free(&exp);
1376
1377 e_sb:
1378         cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1379
1380         return ret;
1381 }
1382
1383 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1384                                 struct ccp_cmd *cmd)
1385 {
1386         struct ccp_passthru_engine *pt = &cmd->u.passthru;
1387         struct ccp_dm_workarea mask;
1388         struct ccp_data src, dst;
1389         struct ccp_op op;
1390         bool in_place = false;
1391         unsigned int i;
1392         int ret = 0;
1393
1394         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1395                 return -EINVAL;
1396
1397         if (!pt->src || !pt->dst)
1398                 return -EINVAL;
1399
1400         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1401                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1402                         return -EINVAL;
1403                 if (!pt->mask)
1404                         return -EINVAL;
1405         }
1406
1407         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1408
1409         memset(&op, 0, sizeof(op));
1410         op.cmd_q = cmd_q;
1411         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1412
1413         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1414                 /* Load the mask */
1415                 op.sb_key = cmd_q->sb_key;
1416
1417                 ret = ccp_init_dm_workarea(&mask, cmd_q,
1418                                            CCP_PASSTHRU_SB_COUNT *
1419                                            CCP_SB_BYTES,
1420                                            DMA_TO_DEVICE);
1421                 if (ret)
1422                         return ret;
1423
1424                 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1425                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1426                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1427                 if (ret) {
1428                         cmd->engine_error = cmd_q->cmd_error;
1429                         goto e_mask;
1430                 }
1431         }
1432
1433         /* Prepare the input and output data workareas. For in-place
1434          * operations we need to set the dma direction to BIDIRECTIONAL
1435          * and copy the src workarea to the dst workarea.
1436          */
1437         if (sg_virt(pt->src) == sg_virt(pt->dst))
1438                 in_place = true;
1439
1440         ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1441                             CCP_PASSTHRU_MASKSIZE,
1442                             in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1443         if (ret)
1444                 goto e_mask;
1445
1446         if (in_place) {
1447                 dst = src;
1448         } else {
1449                 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1450                                     CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1451                 if (ret)
1452                         goto e_src;
1453         }
1454
1455         /* Send data to the CCP Passthru engine
1456          *   Because the CCP engine works on a single source and destination
1457          *   dma address at a time, each entry in the source scatterlist
1458          *   (after the dma_map_sg call) must be less than or equal to the
1459          *   (remaining) length in the destination scatterlist entry and the
1460          *   length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1461          */
1462         dst.sg_wa.sg_used = 0;
1463         for (i = 1; i <= src.sg_wa.dma_count; i++) {
1464                 if (!dst.sg_wa.sg ||
1465                     (sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
1466                         ret = -EINVAL;
1467                         goto e_dst;
1468                 }
1469
1470                 if (i == src.sg_wa.dma_count) {
1471                         op.eom = 1;
1472                         op.soc = 1;
1473                 }
1474
1475                 op.src.type = CCP_MEMTYPE_SYSTEM;
1476                 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1477                 op.src.u.dma.offset = 0;
1478                 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1479
1480                 op.dst.type = CCP_MEMTYPE_SYSTEM;
1481                 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1482                 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1483                 op.dst.u.dma.length = op.src.u.dma.length;
1484
1485                 ret = cmd_q->ccp->vdata->perform->passthru(&op);
1486                 if (ret) {
1487                         cmd->engine_error = cmd_q->cmd_error;
1488                         goto e_dst;
1489                 }
1490
1491                 dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
1492                 if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
1493                         dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1494                         dst.sg_wa.sg_used = 0;
1495                 }
1496                 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1497         }
1498
1499 e_dst:
1500         if (!in_place)
1501                 ccp_free_data(&dst, cmd_q);
1502
1503 e_src:
1504         ccp_free_data(&src, cmd_q);
1505
1506 e_mask:
1507         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1508                 ccp_dm_free(&mask);
1509
1510         return ret;
1511 }
1512
1513 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
1514                                       struct ccp_cmd *cmd)
1515 {
1516         struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
1517         struct ccp_dm_workarea mask;
1518         struct ccp_op op;
1519         int ret;
1520
1521         if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1522                 return -EINVAL;
1523
1524         if (!pt->src_dma || !pt->dst_dma)
1525                 return -EINVAL;
1526
1527         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1528                 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1529                         return -EINVAL;
1530                 if (!pt->mask)
1531                         return -EINVAL;
1532         }
1533
1534         BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1535
1536         memset(&op, 0, sizeof(op));
1537         op.cmd_q = cmd_q;
1538         op.jobid = ccp_gen_jobid(cmd_q->ccp);
1539
1540         if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1541                 /* Load the mask */
1542                 op.sb_key = cmd_q->sb_key;
1543
1544                 mask.length = pt->mask_len;
1545                 mask.dma.address = pt->mask;
1546                 mask.dma.length = pt->mask_len;
1547
1548                 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1549                                      CCP_PASSTHRU_BYTESWAP_NOOP);
1550                 if (ret) {
1551                         cmd->engine_error = cmd_q->cmd_error;
1552                         return ret;
1553                 }
1554         }
1555
1556         /* Send data to the CCP Passthru engine */
1557         op.eom = 1;
1558         op.soc = 1;
1559
1560         op.src.type = CCP_MEMTYPE_SYSTEM;
1561         op.src.u.dma.address = pt->src_dma;
1562         op.src.u.dma.offset = 0;
1563         op.src.u.dma.length = pt->src_len;
1564
1565         op.dst.type = CCP_MEMTYPE_SYSTEM;
1566         op.dst.u.dma.address = pt->dst_dma;
1567         op.dst.u.dma.offset = 0;
1568         op.dst.u.dma.length = pt->src_len;
1569
1570         ret = cmd_q->ccp->vdata->perform->passthru(&op);
1571         if (ret)
1572                 cmd->engine_error = cmd_q->cmd_error;
1573
1574         return ret;
1575 }
1576
1577 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1578 {
1579         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1580         struct ccp_dm_workarea src, dst;
1581         struct ccp_op op;
1582         int ret;
1583         u8 *save;
1584
1585         if (!ecc->u.mm.operand_1 ||
1586             (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1587                 return -EINVAL;
1588
1589         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1590                 if (!ecc->u.mm.operand_2 ||
1591                     (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1592                         return -EINVAL;
1593
1594         if (!ecc->u.mm.result ||
1595             (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1596                 return -EINVAL;
1597
1598         memset(&op, 0, sizeof(op));
1599         op.cmd_q = cmd_q;
1600         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1601
1602         /* Concatenate the modulus and the operands. Both the modulus and
1603          * the operands must be in little endian format.  Since the input
1604          * is in big endian format it must be converted and placed in a
1605          * fixed length buffer.
1606          */
1607         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1608                                    DMA_TO_DEVICE);
1609         if (ret)
1610                 return ret;
1611
1612         /* Save the workarea address since it is updated in order to perform
1613          * the concatenation
1614          */
1615         save = src.address;
1616
1617         /* Copy the ECC modulus */
1618         ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1619                                       CCP_ECC_OPERAND_SIZE, false);
1620         if (ret)
1621                 goto e_src;
1622         src.address += CCP_ECC_OPERAND_SIZE;
1623
1624         /* Copy the first operand */
1625         ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1626                                       ecc->u.mm.operand_1_len,
1627                                       CCP_ECC_OPERAND_SIZE, false);
1628         if (ret)
1629                 goto e_src;
1630         src.address += CCP_ECC_OPERAND_SIZE;
1631
1632         if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1633                 /* Copy the second operand */
1634                 ret = ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1635                                               ecc->u.mm.operand_2_len,
1636                                               CCP_ECC_OPERAND_SIZE, false);
1637                 if (ret)
1638                         goto e_src;
1639                 src.address += CCP_ECC_OPERAND_SIZE;
1640         }
1641
1642         /* Restore the workarea address */
1643         src.address = save;
1644
1645         /* Prepare the output area for the operation */
1646         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1647                                    DMA_FROM_DEVICE);
1648         if (ret)
1649                 goto e_src;
1650
1651         op.soc = 1;
1652         op.src.u.dma.address = src.dma.address;
1653         op.src.u.dma.offset = 0;
1654         op.src.u.dma.length = src.length;
1655         op.dst.u.dma.address = dst.dma.address;
1656         op.dst.u.dma.offset = 0;
1657         op.dst.u.dma.length = dst.length;
1658
1659         op.u.ecc.function = cmd->u.ecc.function;
1660
1661         ret = cmd_q->ccp->vdata->perform->ecc(&op);
1662         if (ret) {
1663                 cmd->engine_error = cmd_q->cmd_error;
1664                 goto e_dst;
1665         }
1666
1667         ecc->ecc_result = le16_to_cpup(
1668                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1669         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1670                 ret = -EIO;
1671                 goto e_dst;
1672         }
1673
1674         /* Save the ECC result */
1675         ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1676
1677 e_dst:
1678         ccp_dm_free(&dst);
1679
1680 e_src:
1681         ccp_dm_free(&src);
1682
1683         return ret;
1684 }
1685
1686 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1687 {
1688         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1689         struct ccp_dm_workarea src, dst;
1690         struct ccp_op op;
1691         int ret;
1692         u8 *save;
1693
1694         if (!ecc->u.pm.point_1.x ||
1695             (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1696             !ecc->u.pm.point_1.y ||
1697             (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1698                 return -EINVAL;
1699
1700         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1701                 if (!ecc->u.pm.point_2.x ||
1702                     (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1703                     !ecc->u.pm.point_2.y ||
1704                     (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1705                         return -EINVAL;
1706         } else {
1707                 if (!ecc->u.pm.domain_a ||
1708                     (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1709                         return -EINVAL;
1710
1711                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1712                         if (!ecc->u.pm.scalar ||
1713                             (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1714                                 return -EINVAL;
1715         }
1716
1717         if (!ecc->u.pm.result.x ||
1718             (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1719             !ecc->u.pm.result.y ||
1720             (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1721                 return -EINVAL;
1722
1723         memset(&op, 0, sizeof(op));
1724         op.cmd_q = cmd_q;
1725         op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1726
1727         /* Concatenate the modulus and the operands. Both the modulus and
1728          * the operands must be in little endian format.  Since the input
1729          * is in big endian format it must be converted and placed in a
1730          * fixed length buffer.
1731          */
1732         ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1733                                    DMA_TO_DEVICE);
1734         if (ret)
1735                 return ret;
1736
1737         /* Save the workarea address since it is updated in order to perform
1738          * the concatenation
1739          */
1740         save = src.address;
1741
1742         /* Copy the ECC modulus */
1743         ret = ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1744                                       CCP_ECC_OPERAND_SIZE, false);
1745         if (ret)
1746                 goto e_src;
1747         src.address += CCP_ECC_OPERAND_SIZE;
1748
1749         /* Copy the first point X and Y coordinate */
1750         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1751                                       ecc->u.pm.point_1.x_len,
1752                                       CCP_ECC_OPERAND_SIZE, false);
1753         if (ret)
1754                 goto e_src;
1755         src.address += CCP_ECC_OPERAND_SIZE;
1756         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1757                                       ecc->u.pm.point_1.y_len,
1758                                       CCP_ECC_OPERAND_SIZE, false);
1759         if (ret)
1760                 goto e_src;
1761         src.address += CCP_ECC_OPERAND_SIZE;
1762
1763         /* Set the first point Z coordinate to 1 */
1764         *src.address = 0x01;
1765         src.address += CCP_ECC_OPERAND_SIZE;
1766
1767         if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1768                 /* Copy the second point X and Y coordinate */
1769                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1770                                               ecc->u.pm.point_2.x_len,
1771                                               CCP_ECC_OPERAND_SIZE, false);
1772                 if (ret)
1773                         goto e_src;
1774                 src.address += CCP_ECC_OPERAND_SIZE;
1775                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1776                                               ecc->u.pm.point_2.y_len,
1777                                               CCP_ECC_OPERAND_SIZE, false);
1778                 if (ret)
1779                         goto e_src;
1780                 src.address += CCP_ECC_OPERAND_SIZE;
1781
1782                 /* Set the second point Z coordinate to 1 */
1783                 *src.address = 0x01;
1784                 src.address += CCP_ECC_OPERAND_SIZE;
1785         } else {
1786                 /* Copy the Domain "a" parameter */
1787                 ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1788                                               ecc->u.pm.domain_a_len,
1789                                               CCP_ECC_OPERAND_SIZE, false);
1790                 if (ret)
1791                         goto e_src;
1792                 src.address += CCP_ECC_OPERAND_SIZE;
1793
1794                 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1795                         /* Copy the scalar value */
1796                         ret = ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
1797                                                       ecc->u.pm.scalar_len,
1798                                                       CCP_ECC_OPERAND_SIZE,
1799                                                       false);
1800                         if (ret)
1801                                 goto e_src;
1802                         src.address += CCP_ECC_OPERAND_SIZE;
1803                 }
1804         }
1805
1806         /* Restore the workarea address */
1807         src.address = save;
1808
1809         /* Prepare the output area for the operation */
1810         ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1811                                    DMA_FROM_DEVICE);
1812         if (ret)
1813                 goto e_src;
1814
1815         op.soc = 1;
1816         op.src.u.dma.address = src.dma.address;
1817         op.src.u.dma.offset = 0;
1818         op.src.u.dma.length = src.length;
1819         op.dst.u.dma.address = dst.dma.address;
1820         op.dst.u.dma.offset = 0;
1821         op.dst.u.dma.length = dst.length;
1822
1823         op.u.ecc.function = cmd->u.ecc.function;
1824
1825         ret = cmd_q->ccp->vdata->perform->ecc(&op);
1826         if (ret) {
1827                 cmd->engine_error = cmd_q->cmd_error;
1828                 goto e_dst;
1829         }
1830
1831         ecc->ecc_result = le16_to_cpup(
1832                 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1833         if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1834                 ret = -EIO;
1835                 goto e_dst;
1836         }
1837
1838         /* Save the workarea address since it is updated as we walk through
1839          * to copy the point math result
1840          */
1841         save = dst.address;
1842
1843         /* Save the ECC result X and Y coordinates */
1844         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
1845                                 CCP_ECC_MODULUS_BYTES);
1846         dst.address += CCP_ECC_OUTPUT_SIZE;
1847         ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
1848                                 CCP_ECC_MODULUS_BYTES);
1849         dst.address += CCP_ECC_OUTPUT_SIZE;
1850
1851         /* Restore the workarea address */
1852         dst.address = save;
1853
1854 e_dst:
1855         ccp_dm_free(&dst);
1856
1857 e_src:
1858         ccp_dm_free(&src);
1859
1860         return ret;
1861 }
1862
1863 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1864 {
1865         struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1866
1867         ecc->ecc_result = 0;
1868
1869         if (!ecc->mod ||
1870             (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
1871                 return -EINVAL;
1872
1873         switch (ecc->function) {
1874         case CCP_ECC_FUNCTION_MMUL_384BIT:
1875         case CCP_ECC_FUNCTION_MADD_384BIT:
1876         case CCP_ECC_FUNCTION_MINV_384BIT:
1877                 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
1878
1879         case CCP_ECC_FUNCTION_PADD_384BIT:
1880         case CCP_ECC_FUNCTION_PMUL_384BIT:
1881         case CCP_ECC_FUNCTION_PDBL_384BIT:
1882                 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
1883
1884         default:
1885                 return -EINVAL;
1886         }
1887 }
1888
1889 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1890 {
1891         int ret;
1892
1893         cmd->engine_error = 0;
1894         cmd_q->cmd_error = 0;
1895         cmd_q->int_rcvd = 0;
1896         cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
1897
1898         switch (cmd->engine) {
1899         case CCP_ENGINE_AES:
1900                 ret = ccp_run_aes_cmd(cmd_q, cmd);
1901                 break;
1902         case CCP_ENGINE_XTS_AES_128:
1903                 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
1904                 break;
1905         case CCP_ENGINE_SHA:
1906                 ret = ccp_run_sha_cmd(cmd_q, cmd);
1907                 break;
1908         case CCP_ENGINE_RSA:
1909                 ret = ccp_run_rsa_cmd(cmd_q, cmd);
1910                 break;
1911         case CCP_ENGINE_PASSTHRU:
1912                 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
1913                         ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
1914                 else
1915                         ret = ccp_run_passthru_cmd(cmd_q, cmd);
1916                 break;
1917         case CCP_ENGINE_ECC:
1918                 ret = ccp_run_ecc_cmd(cmd_q, cmd);
1919                 break;
1920         default:
1921                 ret = -EINVAL;
1922         }
1923
1924         return ret;
1925 }