drivers/crypto/ccp/ccp-dev-v5.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Gary R Hook <gary.hook@amd.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/kernel.h>
  15 #include <linux/pci.h>
  16 #include <linux/kthread.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/compiler.h>
  20 #include <linux/ccp.h>
  21
  22 #include "ccp-dev.h"
  23
  24 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
  25 {
  26         struct ccp_device *ccp;
  27         int start;
  28
  29         /* First look at the map for the queue */
  30         if (cmd_q->lsb >= 0) {
  31                 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
  32                                                         LSB_SIZE,
  33                                                         0, count, 0);
  34                 if (start < LSB_SIZE) {
  35                         bitmap_set(cmd_q->lsbmap, start, count);
  36                         return start + cmd_q->lsb * LSB_SIZE;
  37                 }
  38         }
  39
  40         /* No joy; try to get an entry from the shared blocks */
  41         ccp = cmd_q->ccp;
  42         for (;;) {
  43                 mutex_lock(&ccp->sb_mutex);
  44
  45                 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
  46                                                         MAX_LSB_CNT * LSB_SIZE,
  47                                                         0,
  48                                                         count, 0);
  49                 if (start <= MAX_LSB_CNT * LSB_SIZE) {
  50                         bitmap_set(ccp->lsbmap, start, count);
  51
  52                         mutex_unlock(&ccp->sb_mutex);
  53                         return start * LSB_ITEM_SIZE;
  54                 }
  55
  56                 ccp->sb_avail = 0;
  57
  58                 mutex_unlock(&ccp->sb_mutex);
  59
  60                 /* Wait for KSB entries to become available */
  61                 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
  62                         return 0;
  63         }
  64 }
  65
  66 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
  67                          unsigned int count)
  68 {
  69         int lsbno = start / LSB_SIZE;
  70
  71         if (!start)
  72                 return;
  73
  74         if (cmd_q->lsb == lsbno) {
  75                 /* An entry from the private LSB */
  76                 bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count);
  77         } else {
  78                 /* From the shared LSBs */
  79                 struct ccp_device *ccp = cmd_q->ccp;
  80
  81                 mutex_lock(&ccp->sb_mutex);
  82                 bitmap_clear(ccp->lsbmap, start, count);
  83                 ccp->sb_avail = 1;
  84                 mutex_unlock(&ccp->sb_mutex);
  85                 wake_up_interruptible_all(&ccp->sb_queue);
  86         }
  87 }
  88
  89 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
  90 union ccp_function {
  91         struct {
  92                 u16 size:7;
  93                 u16 encrypt:1;
  94                 u16 mode:5;
  95                 u16 type:2;
  96         } aes;
  97         struct {
  98                 u16 size:7;
  99                 u16 encrypt:1;
 100                 u16 rsvd:5;
 101                 u16 type:2;
 102         } aes_xts;
 103         struct {
 104                 u16 rsvd1:10;
 105                 u16 type:4;
 106                 u16 rsvd2:1;
 107         } sha;
 108         struct {
 109                 u16 mode:3;
 110                 u16 size:12;
 111         } rsa;
 112         struct {
 113                 u16 byteswap:2;
 114                 u16 bitwise:3;
 115                 u16 reflect:2;
 116                 u16 rsvd:8;
 117         } pt;
 118         struct  {
 119                 u16 rsvd:13;
 120         } zlib;
 121         struct {
 122                 u16 size:10;
 123                 u16 type:2;
 124                 u16 mode:3;
 125         } ecc;
 126         u16 raw;
 127 };
 128
 129 #define CCP_AES_SIZE(p)         ((p)->aes.size)
 130 #define CCP_AES_ENCRYPT(p)      ((p)->aes.encrypt)
 131 #define CCP_AES_MODE(p)         ((p)->aes.mode)
 132 #define CCP_AES_TYPE(p)         ((p)->aes.type)
 133 #define CCP_XTS_SIZE(p)         ((p)->aes_xts.size)
 134 #define CCP_XTS_TYPE(p)         ((p)->aes_xts.type)
 135 #define CCP_XTS_ENCRYPT(p)      ((p)->aes_xts.encrypt)
 136 #define CCP_SHA_TYPE(p)         ((p)->sha.type)
 137 #define CCP_RSA_SIZE(p)         ((p)->rsa.size)
 138 #define CCP_PT_BYTESWAP(p)      ((p)->pt.byteswap)
 139 #define CCP_PT_BITWISE(p)       ((p)->pt.bitwise)
 140 #define CCP_ECC_MODE(p)         ((p)->ecc.mode)
 141 #define CCP_ECC_AFFINE(p)       ((p)->ecc.one)
 142
 143 /* Word 0 */
 144 #define CCP5_CMD_DW0(p)         ((p)->dw0)
 145 #define CCP5_CMD_SOC(p)         (CCP5_CMD_DW0(p).soc)
 146 #define CCP5_CMD_IOC(p)         (CCP5_CMD_DW0(p).ioc)
 147 #define CCP5_CMD_INIT(p)        (CCP5_CMD_DW0(p).init)
 148 #define CCP5_CMD_EOM(p)         (CCP5_CMD_DW0(p).eom)
 149 #define CCP5_CMD_FUNCTION(p)    (CCP5_CMD_DW0(p).function)
 150 #define CCP5_CMD_ENGINE(p)      (CCP5_CMD_DW0(p).engine)
 151 #define CCP5_CMD_PROT(p)        (CCP5_CMD_DW0(p).prot)
 152
 153 /* Word 1 */
 154 #define CCP5_CMD_DW1(p)         ((p)->length)
 155 #define CCP5_CMD_LEN(p)         (CCP5_CMD_DW1(p))
 156
 157 /* Word 2 */
 158 #define CCP5_CMD_DW2(p)         ((p)->src_lo)
 159 #define CCP5_CMD_SRC_LO(p)      (CCP5_CMD_DW2(p))
 160
 161 /* Word 3 */
 162 #define CCP5_CMD_DW3(p)         ((p)->dw3)
 163 #define CCP5_CMD_SRC_MEM(p)     ((p)->dw3.src_mem)
 164 #define CCP5_CMD_SRC_HI(p)      ((p)->dw3.src_hi)
 165 #define CCP5_CMD_LSB_ID(p)      ((p)->dw3.lsb_cxt_id)
 166 #define CCP5_CMD_FIX_SRC(p)     ((p)->dw3.fixed)
 167
 168 /* Words 4/5 */
 169 #define CCP5_CMD_DW4(p)         ((p)->dw4)
 170 #define CCP5_CMD_DST_LO(p)      (CCP5_CMD_DW4(p).dst_lo)
 171 #define CCP5_CMD_DW5(p)         ((p)->dw5.fields.dst_hi)
 172 #define CCP5_CMD_DST_HI(p)      (CCP5_CMD_DW5(p))
 173 #define CCP5_CMD_DST_MEM(p)     ((p)->dw5.fields.dst_mem)
 174 #define CCP5_CMD_FIX_DST(p)     ((p)->dw5.fields.fixed)
 175 #define CCP5_CMD_SHA_LO(p)      ((p)->dw4.sha_len_lo)
 176 #define CCP5_CMD_SHA_HI(p)      ((p)->dw5.sha_len_hi)
 177
 178 /* Word 6/7 */
 179 #define CCP5_CMD_DW6(p)         ((p)->key_lo)
 180 #define CCP5_CMD_KEY_LO(p)      (CCP5_CMD_DW6(p))
 181 #define CCP5_CMD_DW7(p)         ((p)->dw7)
 182 #define CCP5_CMD_KEY_HI(p)      ((p)->dw7.key_hi)
 183 #define CCP5_CMD_KEY_MEM(p)     ((p)->dw7.key_mem)
 184
 185 static inline u32 low_address(unsigned long addr)
 186 {
 187         return (u64)addr & 0x0ffffffff;
 188 }
 189
 190 static inline u32 high_address(unsigned long addr)
 191 {
 192         return ((u64)addr >> 32) & 0x00000ffff;
 193 }
 194
 195 static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
 196 {
 197         unsigned int head_idx, n;
 198         u32 head_lo, queue_start;
 199
 200         queue_start = low_address(cmd_q->qdma_tail);
 201         head_lo = ioread32(cmd_q->reg_head_lo);
 202         head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
 203
 204         n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
 205
 206         return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
 207 }
 208
 209 static int ccp5_do_cmd(struct ccp5_desc *desc,
 210                        struct ccp_cmd_queue *cmd_q)
 211 {
 212         u32 *mP;
 213         __le32 *dP;
 214         u32 tail;
 215         int     i;
 216         int ret = 0;
 217
 218         if (CCP5_CMD_SOC(desc)) {
 219                 CCP5_CMD_IOC(desc) = 1;
 220                 CCP5_CMD_SOC(desc) = 0;
 221         }
 222         mutex_lock(&cmd_q->q_mutex);
 223
 224         mP = (u32 *) &cmd_q->qbase[cmd_q->qidx];
 225         dP = (__le32 *) desc;
 226         for (i = 0; i < 8; i++)
 227                 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
 228
 229         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
 230
 231         /* The data used by this command must be flushed to memory */
 232         wmb();
 233
 234         /* Write the new tail address back to the queue register */
 235         tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
 236         iowrite32(tail, cmd_q->reg_tail_lo);
 237
 238         /* Turn the queue back on using our cached control register */
 239         iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
 240         mutex_unlock(&cmd_q->q_mutex);
 241
 242         if (CCP5_CMD_IOC(desc)) {
 243                 /* Wait for the job to complete */
 244                 ret = wait_event_interruptible(cmd_q->int_queue,
 245                                                cmd_q->int_rcvd);
 246                 if (ret || cmd_q->cmd_error) {
 247                         if (cmd_q->cmd_error)
 248                                 ccp_log_error(cmd_q->ccp,
 249                                               cmd_q->cmd_error);
 250                         /* A version 5 device doesn't use Job IDs... */
 251                         if (!ret)
 252                                 ret = -EIO;
 253                 }
 254                 cmd_q->int_rcvd = 0;
 255         }
 256
 257         return 0;
 258 }
 259
 260 static int ccp5_perform_aes(struct ccp_op *op)
 261 {
 262         struct ccp5_desc desc;
 263         union ccp_function function;
 264         u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
 265
 266         /* Zero out all the fields of the command desc */
 267         memset(&desc, 0, Q_DESC_SIZE);
 268
 269         CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
 270
 271         CCP5_CMD_SOC(&desc) = op->soc;
 272         CCP5_CMD_IOC(&desc) = 1;
 273         CCP5_CMD_INIT(&desc) = op->init;
 274         CCP5_CMD_EOM(&desc) = op->eom;
 275         CCP5_CMD_PROT(&desc) = 0;
 276
 277         function.raw = 0;
 278         CCP_AES_ENCRYPT(&function) = op->u.aes.action;
 279         CCP_AES_MODE(&function) = op->u.aes.mode;
 280         CCP_AES_TYPE(&function) = op->u.aes.type;
 281         CCP_AES_SIZE(&function) = op->u.aes.size;
 282
 283         CCP5_CMD_FUNCTION(&desc) = function.raw;
 284
 285         CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
 286
 287         CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
 288         CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
 289         CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 290
 291         CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
 292         CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
 293         CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 294
 295         CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
 296         CCP5_CMD_KEY_HI(&desc) = 0;
 297         CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
 298         CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
 299
 300         return ccp5_do_cmd(&desc, op->cmd_q);
 301 }
 302
 303 static int ccp5_perform_xts_aes(struct ccp_op *op)
 304 {
 305         struct ccp5_desc desc;
 306         union ccp_function function;
 307         u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
 308
 309         /* Zero out all the fields of the command desc */
 310         memset(&desc, 0, Q_DESC_SIZE);
 311
 312         CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
 313
 314         CCP5_CMD_SOC(&desc) = op->soc;
 315         CCP5_CMD_IOC(&desc) = 1;
 316         CCP5_CMD_INIT(&desc) = op->init;
 317         CCP5_CMD_EOM(&desc) = op->eom;
 318         CCP5_CMD_PROT(&desc) = 0;
 319
 320         function.raw = 0;
 321         CCP_XTS_TYPE(&function) = op->u.xts.type;
 322         CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
 323         CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
 324         CCP5_CMD_FUNCTION(&desc) = function.raw;
 325
 326         CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
 327
 328         CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
 329         CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
 330         CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 331
 332         CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
 333         CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
 334         CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 335
 336         CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
 337         CCP5_CMD_KEY_HI(&desc) =  0;
 338         CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
 339         CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
 340
 341         return ccp5_do_cmd(&desc, op->cmd_q);
 342 }
 343
 344 static int ccp5_perform_sha(struct ccp_op *op)
 345 {
 346         struct ccp5_desc desc;
 347         union ccp_function function;
 348
 349         /* Zero out all the fields of the command desc */
 350         memset(&desc, 0, Q_DESC_SIZE);
 351
 352         CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
 353
 354         CCP5_CMD_SOC(&desc) = op->soc;
 355         CCP5_CMD_IOC(&desc) = 1;
 356         CCP5_CMD_INIT(&desc) = 1;
 357         CCP5_CMD_EOM(&desc) = op->eom;
 358         CCP5_CMD_PROT(&desc) = 0;
 359
 360         function.raw = 0;
 361         CCP_SHA_TYPE(&function) = op->u.sha.type;
 362         CCP5_CMD_FUNCTION(&desc) = function.raw;
 363
 364         CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
 365
 366         CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
 367         CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
 368         CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 369
 370         CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
 371
 372         if (op->eom) {
 373                 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
 374                 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
 375         } else {
 376                 CCP5_CMD_SHA_LO(&desc) = 0;
 377                 CCP5_CMD_SHA_HI(&desc) = 0;
 378         }
 379
 380         return ccp5_do_cmd(&desc, op->cmd_q);
 381 }
 382
 383 static int ccp5_perform_rsa(struct ccp_op *op)
 384 {
 385         struct ccp5_desc desc;
 386         union ccp_function function;
 387
 388         /* Zero out all the fields of the command desc */
 389         memset(&desc, 0, Q_DESC_SIZE);
 390
 391         CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
 392
 393         CCP5_CMD_SOC(&desc) = op->soc;
 394         CCP5_CMD_IOC(&desc) = 1;
 395         CCP5_CMD_INIT(&desc) = 0;
 396         CCP5_CMD_EOM(&desc) = 1;
 397         CCP5_CMD_PROT(&desc) = 0;
 398
 399         function.raw = 0;
 400         CCP_RSA_SIZE(&function) = op->u.rsa.mod_size;
 401         CCP5_CMD_FUNCTION(&desc) = function.raw;
 402
 403         CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
 404
 405         /* Source is from external memory */
 406         CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
 407         CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
 408         CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 409
 410         /* Destination is in external memory */
 411         CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
 412         CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
 413         CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 414
 415         /* Key (Exponent) is in external memory */
 416         CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
 417         CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
 418         CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 419
 420         return ccp5_do_cmd(&desc, op->cmd_q);
 421 }
 422
 423 static int ccp5_perform_passthru(struct ccp_op *op)
 424 {
 425         struct ccp5_desc desc;
 426         union ccp_function function;
 427         struct ccp_dma_info *saddr = &op->src.u.dma;
 428         struct ccp_dma_info *daddr = &op->dst.u.dma;
 429
 430         memset(&desc, 0, Q_DESC_SIZE);
 431
 432         CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
 433
 434         CCP5_CMD_SOC(&desc) = 0;
 435         CCP5_CMD_IOC(&desc) = 1;
 436         CCP5_CMD_INIT(&desc) = 0;
 437         CCP5_CMD_EOM(&desc) = op->eom;
 438         CCP5_CMD_PROT(&desc) = 0;
 439
 440         function.raw = 0;
 441         CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
 442         CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
 443         CCP5_CMD_FUNCTION(&desc) = function.raw;
 444
 445         /* Length of source data is always 256 bytes */
 446         if (op->src.type == CCP_MEMTYPE_SYSTEM)
 447                 CCP5_CMD_LEN(&desc) = saddr->length;
 448         else
 449                 CCP5_CMD_LEN(&desc) = daddr->length;
 450
 451         if (op->src.type == CCP_MEMTYPE_SYSTEM) {
 452                 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
 453                 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
 454                 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 455
 456                 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
 457                         CCP5_CMD_LSB_ID(&desc) = op->sb_key;
 458         } else {
 459                 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
 460
 461                 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
 462                 CCP5_CMD_SRC_HI(&desc) = 0;
 463                 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
 464         }
 465
 466         if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
 467                 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
 468                 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
 469                 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 470         } else {
 471                 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
 472
 473                 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
 474                 CCP5_CMD_DST_HI(&desc) = 0;
 475                 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
 476         }
 477
 478         return ccp5_do_cmd(&desc, op->cmd_q);
 479 }
 480
 481 static int ccp5_perform_ecc(struct ccp_op *op)
 482 {
 483         struct ccp5_desc desc;
 484         union ccp_function function;
 485
 486         /* Zero out all the fields of the command desc */
 487         memset(&desc, 0, Q_DESC_SIZE);
 488
 489         CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
 490
 491         CCP5_CMD_SOC(&desc) = 0;
 492         CCP5_CMD_IOC(&desc) = 1;
 493         CCP5_CMD_INIT(&desc) = 0;
 494         CCP5_CMD_EOM(&desc) = 1;
 495         CCP5_CMD_PROT(&desc) = 0;
 496
 497         function.raw = 0;
 498         function.ecc.mode = op->u.ecc.function;
 499         CCP5_CMD_FUNCTION(&desc) = function.raw;
 500
 501         CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
 502
 503         CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
 504         CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
 505         CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 506
 507         CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
 508         CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
 509         CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
 510
 511         return ccp5_do_cmd(&desc, op->cmd_q);
 512 }
 513
 514 static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
 515 {
 516         int q_mask = 1 << cmd_q->id;
 517         int queues = 0;
 518         int j;
 519
 520         /* Build a bit mask to know which LSBs this queue has access to.
 521          * Don't bother with segment 0 as it has special privileges.
 522          */
 523         for (j = 1; j < MAX_LSB_CNT; j++) {
 524                 if (status & q_mask)
 525                         bitmap_set(cmd_q->lsbmask, j, 1);
 526                 status >>= LSB_REGION_WIDTH;
 527         }
 528         queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
 529         dev_info(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
 530                  cmd_q->id, queues);
 531
 532         return queues ? 0 : -EINVAL;
 533 }
 534
 535
 536 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
 537                                         int lsb_cnt, int n_lsbs,
 538                                         unsigned long *lsb_pub)
 539 {
 540         DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
 541         int bitno;
 542         int qlsb_wgt;
 543         int i;
 544
 545         /* For each queue:
 546          * If the count of potential LSBs available to a queue matches the
 547          * ordinal given to us in lsb_cnt:
 548          * Copy the mask of possible LSBs for this queue into "qlsb";
 549          * For each bit in qlsb, see if the corresponding bit in the
 550          * aggregation mask is set; if so, we have a match.
 551          *     If we have a match, clear the bit in the aggregation to
 552          *     mark it as no longer available.
 553          *     If there is no match, clear the bit in qlsb and keep looking.
 554          */
 555         for (i = 0; i < ccp->cmd_q_count; i++) {
 556                 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
 557
 558                 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
 559
 560                 if (qlsb_wgt == lsb_cnt) {
 561                         bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
 562
 563                         bitno = find_first_bit(qlsb, MAX_LSB_CNT);
 564                         while (bitno < MAX_LSB_CNT) {
 565                                 if (test_bit(bitno, lsb_pub)) {
 566                                         /* We found an available LSB
 567                                          * that this queue can access
 568                                          */
 569                                         cmd_q->lsb = bitno;
 570                                         bitmap_clear(lsb_pub, bitno, 1);
 571                                         dev_info(ccp->dev,
 572                                                  "Queue %d gets LSB %d\n",
 573                                                  i, bitno);
 574                                         break;
 575                                 }
 576                                 bitmap_clear(qlsb, bitno, 1);
 577                                 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
 578                         }
 579                         if (bitno >= MAX_LSB_CNT)
 580                                 return -EINVAL;
 581                         n_lsbs--;
 582                 }
 583         }
 584         return n_lsbs;
 585 }
 586
 587 /* For each queue, from the most- to least-constrained:
 588  * find an LSB that can be assigned to the queue. If there are N queues that
 589  * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
 590  * dedicated LSB. Remaining LSB regions become a shared resource.
 591  * If we have fewer LSBs than queues, all LSB regions become shared resources.
 592  */
 593 static int ccp_assign_lsbs(struct ccp_device *ccp)
 594 {
 595         DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
 596         DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
 597         int n_lsbs = 0;
 598         int bitno;
 599         int i, lsb_cnt;
 600         int rc = 0;
 601
 602         bitmap_zero(lsb_pub, MAX_LSB_CNT);
 603
 604         /* Create an aggregate bitmap to get a total count of available LSBs */
 605         for (i = 0; i < ccp->cmd_q_count; i++)
 606                 bitmap_or(lsb_pub,
 607                           lsb_pub, ccp->cmd_q[i].lsbmask,
 608                           MAX_LSB_CNT);
 609
 610         n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
 611
 612         if (n_lsbs >= ccp->cmd_q_count) {
 613                 /* We have enough LSBS to give every queue a private LSB.
 614                  * Brute force search to start with the queues that are more
 615                  * constrained in LSB choice. When an LSB is privately
 616                  * assigned, it is removed from the public mask.
 617                  * This is an ugly N squared algorithm with some optimization.
 618                  */
 619                 for (lsb_cnt = 1;
 620                      n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
 621                      lsb_cnt++) {
 622                         rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
 623                                                           lsb_pub);
 624                         if (rc < 0)
 625                                 return -EINVAL;
 626                         n_lsbs = rc;
 627                 }
 628         }
 629
 630         rc = 0;
 631         /* What's left of the LSBs, according to the public mask, now become
 632          * shared. Any zero bits in the lsb_pub mask represent an LSB region
 633          * that can't be used as a shared resource, so mark the LSB slots for
 634          * them as "in use".
 635          */
 636         bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
 637
 638         bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
 639         while (bitno < MAX_LSB_CNT) {
 640                 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
 641                 bitmap_set(qlsb, bitno, 1);
 642                 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
 643         }
 644
 645         return rc;
 646 }
 647
 648 static void ccp5_disable_queue_interrupts(struct ccp_device *ccp)
 649 {
 650         unsigned int i;
 651
 652         for (i = 0; i < ccp->cmd_q_count; i++)
 653                 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable);
 654 }
 655
 656 static void ccp5_enable_queue_interrupts(struct ccp_device *ccp)
 657 {
 658         unsigned int i;
 659
 660         for (i = 0; i < ccp->cmd_q_count; i++)
 661                 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable);
 662 }
 663
 664 static void ccp5_irq_bh(unsigned long data)
 665 {
 666         struct ccp_device *ccp = (struct ccp_device *)data;
 667         u32 status;
 668         unsigned int i;
 669
 670         for (i = 0; i < ccp->cmd_q_count; i++) {
 671                 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
 672
 673                 status = ioread32(cmd_q->reg_interrupt_status);
 674
 675                 if (status) {
 676                         cmd_q->int_status = status;
 677                         cmd_q->q_status = ioread32(cmd_q->reg_status);
 678                         cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
 679
 680                         /* On error, only save the first error value */
 681                         if ((status & INT_ERROR) && !cmd_q->cmd_error)
 682                                 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
 683
 684                         cmd_q->int_rcvd = 1;
 685
 686                         /* Acknowledge the interrupt and wake the kthread */
 687                         iowrite32(status, cmd_q->reg_interrupt_status);
 688                         wake_up_interruptible(&cmd_q->int_queue);
 689                 }
 690         }
 691         ccp5_enable_queue_interrupts(ccp);
 692 }
 693
 694 static irqreturn_t ccp5_irq_handler(int irq, void *data)
 695 {
 696         struct device *dev = data;
 697         struct ccp_device *ccp = dev_get_drvdata(dev);
 698
 699         ccp5_disable_queue_interrupts(ccp);
 700         if (ccp->use_tasklet)
 701                 tasklet_schedule(&ccp->irq_tasklet);
 702         else
 703                 ccp5_irq_bh((unsigned long)ccp);
 704         return IRQ_HANDLED;
 705 }
 706
 707 static int ccp5_init(struct ccp_device *ccp)
 708 {
 709         struct device *dev = ccp->dev;
 710         struct ccp_cmd_queue *cmd_q;
 711         struct dma_pool *dma_pool;
 712         char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
 713         unsigned int qmr, qim, i;
 714         u64 status;
 715         u32 status_lo, status_hi;
 716         int ret;
 717
 718         /* Find available queues */
 719         qim = 0;
 720         qmr = ioread32(ccp->io_regs + Q_MASK_REG);
 721         for (i = 0; i < MAX_HW_QUEUES; i++) {
 722
 723                 if (!(qmr & (1 << i)))
 724                         continue;
 725
 726                 /* Allocate a dma pool for this queue */
 727                 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
 728                          ccp->name, i);
 729                 dma_pool = dma_pool_create(dma_pool_name, dev,
 730                                            CCP_DMAPOOL_MAX_SIZE,
 731                                            CCP_DMAPOOL_ALIGN, 0);
 732                 if (!dma_pool) {
 733                         dev_err(dev, "unable to allocate dma pool\n");
 734                         ret = -ENOMEM;
 735                 }
 736
 737                 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
 738                 ccp->cmd_q_count++;
 739
 740                 cmd_q->ccp = ccp;
 741                 cmd_q->id = i;
 742                 cmd_q->dma_pool = dma_pool;
 743                 mutex_init(&cmd_q->q_mutex);
 744
 745                 /* Page alignment satisfies our needs for N <= 128 */
 746                 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
 747                 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
 748                 cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize,
 749                                                    &cmd_q->qbase_dma,
 750                                                    GFP_KERNEL);
 751                 if (!cmd_q->qbase) {
 752                         dev_err(dev, "unable to allocate command queue\n");
 753                         ret = -ENOMEM;
 754                         goto e_pool;
 755                 }
 756
 757                 cmd_q->qidx = 0;
 758                 /* Preset some register values and masks that are queue
 759                  * number dependent
 760                  */
 761                 cmd_q->reg_control = ccp->io_regs +
 762                                      CMD5_Q_STATUS_INCR * (i + 1);
 763                 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
 764                 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
 765                 cmd_q->reg_int_enable = cmd_q->reg_control +
 766                                         CMD5_Q_INT_ENABLE_BASE;
 767                 cmd_q->reg_interrupt_status = cmd_q->reg_control +
 768                                               CMD5_Q_INTERRUPT_STATUS_BASE;
 769                 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
 770                 cmd_q->reg_int_status = cmd_q->reg_control +
 771                                         CMD5_Q_INT_STATUS_BASE;
 772                 cmd_q->reg_dma_status = cmd_q->reg_control +
 773                                         CMD5_Q_DMA_STATUS_BASE;
 774                 cmd_q->reg_dma_read_status = cmd_q->reg_control +
 775                                              CMD5_Q_DMA_READ_STATUS_BASE;
 776                 cmd_q->reg_dma_write_status = cmd_q->reg_control +
 777                                               CMD5_Q_DMA_WRITE_STATUS_BASE;
 778
 779                 init_waitqueue_head(&cmd_q->int_queue);
 780
 781                 dev_dbg(dev, "queue #%u available\n", i);
 782         }
 783         if (ccp->cmd_q_count == 0) {
 784                 dev_notice(dev, "no command queues available\n");
 785                 ret = -EIO;
 786                 goto e_pool;
 787         }
 788         dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
 789
 790         /* Turn off the queues and disable interrupts until ready */
 791         ccp5_disable_queue_interrupts(ccp);
 792         for (i = 0; i < ccp->cmd_q_count; i++) {
 793                 cmd_q = &ccp->cmd_q[i];
 794
 795                 cmd_q->qcontrol = 0; /* Start with nothing */
 796                 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
 797
 798                 ioread32(cmd_q->reg_int_status);
 799                 ioread32(cmd_q->reg_status);
 800
 801                 /* Clear the interrupt status */
 802                 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
 803         }
 804
 805         dev_dbg(dev, "Requesting an IRQ...\n");
 806         /* Request an irq */
 807         ret = ccp->get_irq(ccp);
 808         if (ret) {
 809                 dev_err(dev, "unable to allocate an IRQ\n");
 810                 goto e_pool;
 811         }
 812         /* Initialize the ISR tasklet */
 813         if (ccp->use_tasklet)
 814                 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh,
 815                              (unsigned long)ccp);
 816
 817         /* Initialize the queue used to suspend */
 818         init_waitqueue_head(&ccp->suspend_queue);
 819
 820         dev_dbg(dev, "Loading LSB map...\n");
 821         /* Copy the private LSB mask to the public registers */
 822         status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
 823         status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
 824         iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
 825         iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
 826         status = ((u64)status_hi<<30) | (u64)status_lo;
 827
 828         dev_dbg(dev, "Configuring virtual queues...\n");
 829         /* Configure size of each virtual queue accessible to host */
 830         for (i = 0; i < ccp->cmd_q_count; i++) {
 831                 u32 dma_addr_lo;
 832                 u32 dma_addr_hi;
 833
 834                 cmd_q = &ccp->cmd_q[i];
 835
 836                 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
 837                 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
 838
 839                 cmd_q->qdma_tail = cmd_q->qbase_dma;
 840                 dma_addr_lo = low_address(cmd_q->qdma_tail);
 841                 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
 842                 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
 843
 844                 dma_addr_hi = high_address(cmd_q->qdma_tail);
 845                 cmd_q->qcontrol |= (dma_addr_hi << 16);
 846                 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
 847
 848                 /* Find the LSB regions accessible to the queue */
 849                 ccp_find_lsb_regions(cmd_q, status);
 850                 cmd_q->lsb = -1; /* Unassigned value */
 851         }
 852
 853         dev_dbg(dev, "Assigning LSBs...\n");
 854         ret = ccp_assign_lsbs(ccp);
 855         if (ret) {
 856                 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
 857                 goto e_irq;
 858         }
 859
 860         /* Optimization: pre-allocate LSB slots for each queue */
 861         for (i = 0; i < ccp->cmd_q_count; i++) {
 862                 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
 863                 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
 864         }
 865
 866         dev_dbg(dev, "Starting threads...\n");
 867         /* Create a kthread for each queue */
 868         for (i = 0; i < ccp->cmd_q_count; i++) {
 869                 struct task_struct *kthread;
 870
 871                 cmd_q = &ccp->cmd_q[i];
 872
 873                 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
 874                                          "%s-q%u", ccp->name, cmd_q->id);
 875                 if (IS_ERR(kthread)) {
 876                         dev_err(dev, "error creating queue thread (%ld)\n",
 877                                 PTR_ERR(kthread));
 878                         ret = PTR_ERR(kthread);
 879                         goto e_kthread;
 880                 }
 881
 882                 cmd_q->kthread = kthread;
 883                 wake_up_process(kthread);
 884         }
 885
 886         dev_dbg(dev, "Enabling interrupts...\n");
 887         ccp5_enable_queue_interrupts(ccp);
 888
 889         dev_dbg(dev, "Registering device...\n");
 890         /* Put this on the unit list to make it available */
 891         ccp_add_device(ccp);
 892
 893         ret = ccp_register_rng(ccp);
 894         if (ret)
 895                 goto e_kthread;
 896
 897         /* Register the DMA engine support */
 898         ret = ccp_dmaengine_register(ccp);
 899         if (ret)
 900                 goto e_hwrng;
 901
 902         return 0;
 903
 904 e_hwrng:
 905         ccp_unregister_rng(ccp);
 906
 907 e_kthread:
 908         for (i = 0; i < ccp->cmd_q_count; i++)
 909                 if (ccp->cmd_q[i].kthread)
 910                         kthread_stop(ccp->cmd_q[i].kthread);
 911
 912 e_irq:
 913         ccp->free_irq(ccp);
 914
 915 e_pool:
 916         for (i = 0; i < ccp->cmd_q_count; i++)
 917                 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
 918
 919         return ret;
 920 }
 921
 922 static void ccp5_destroy(struct ccp_device *ccp)
 923 {
 924         struct device *dev = ccp->dev;
 925         struct ccp_cmd_queue *cmd_q;
 926         struct ccp_cmd *cmd;
 927         unsigned int i;
 928
 929         /* Unregister the DMA engine */
 930         ccp_dmaengine_unregister(ccp);
 931
 932         /* Unregister the RNG */
 933         ccp_unregister_rng(ccp);
 934
 935         /* Remove this device from the list of available units first */
 936         ccp_del_device(ccp);
 937
 938         /* Disable and clear interrupts */
 939         ccp5_disable_queue_interrupts(ccp);
 940         for (i = 0; i < ccp->cmd_q_count; i++) {
 941                 cmd_q = &ccp->cmd_q[i];
 942
 943                 /* Turn off the run bit */
 944                 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
 945
 946                 /* Clear the interrupt status */
 947                 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
 948                 ioread32(cmd_q->reg_int_status);
 949                 ioread32(cmd_q->reg_status);
 950         }
 951
 952         /* Stop the queue kthreads */
 953         for (i = 0; i < ccp->cmd_q_count; i++)
 954                 if (ccp->cmd_q[i].kthread)
 955                         kthread_stop(ccp->cmd_q[i].kthread);
 956
 957         ccp->free_irq(ccp);
 958
 959         for (i = 0; i < ccp->cmd_q_count; i++) {
 960                 cmd_q = &ccp->cmd_q[i];
 961                 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
 962                                   cmd_q->qbase_dma);
 963         }
 964
 965         /* Flush the cmd and backlog queue */
 966         while (!list_empty(&ccp->cmd)) {
 967                 /* Invoke the callback directly with an error code */
 968                 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
 969                 list_del(&cmd->entry);
 970                 cmd->callback(cmd->data, -ENODEV);
 971         }
 972         while (!list_empty(&ccp->backlog)) {
 973                 /* Invoke the callback directly with an error code */
 974                 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
 975                 list_del(&cmd->entry);
 976                 cmd->callback(cmd->data, -ENODEV);
 977         }
 978 }
 979
 980 static void ccp5_config(struct ccp_device *ccp)
 981 {
 982         /* Public side */
 983         iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
 984 }
 985
 986 static void ccp5other_config(struct ccp_device *ccp)
 987 {
 988         int i;
 989         u32 rnd;
 990
 991         /* We own all of the queues on the NTB CCP */
 992
 993         iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
 994         iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
 995         for (i = 0; i < 12; i++) {
 996                 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
 997                 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
 998         }
 999
1000         iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
1001         iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
1002         iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
1003
1004         iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
1005         iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
1006
1007         iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
1008
1009         ccp5_config(ccp);
1010 }
1011
1012 /* Version 5 adds some function, but is essentially the same as v5 */
1013 static const struct ccp_actions ccp5_actions = {
1014         .aes = ccp5_perform_aes,
1015         .xts_aes = ccp5_perform_xts_aes,
1016         .sha = ccp5_perform_sha,
1017         .rsa = ccp5_perform_rsa,
1018         .passthru = ccp5_perform_passthru,
1019         .ecc = ccp5_perform_ecc,
1020         .sballoc = ccp_lsb_alloc,
1021         .sbfree = ccp_lsb_free,
1022         .init = ccp5_init,
1023         .destroy = ccp5_destroy,
1024         .get_free_slots = ccp5_get_free_slots,
1025         .irqhandler = ccp5_irq_handler,
1026 };
1027
1028 const struct ccp_vdata ccpv5a = {
1029         .version = CCP_VERSION(5, 0),
1030         .setup = ccp5_config,
1031         .perform = &ccp5_actions,
1032         .bar = 2,
1033         .offset = 0x0,
1034 };
1035
1036 const struct ccp_vdata ccpv5b = {
1037         .version = CCP_VERSION(5, 0),
1038         .setup = ccp5other_config,
1039         .perform = &ccp5_actions,
1040         .bar = 2,
1041         .offset = 0x0,
1042 };