2 * AMD Cryptographic Coprocessor (CCP) driver
4 * Copyright (C) 2016 Advanced Micro Devices, Inc.
6 * Author: Gary R Hook <gary.hook@amd.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #include <linux/module.h>
14 #include <linux/kernel.h>
15 #include <linux/pci.h>
16 #include <linux/kthread.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/interrupt.h>
19 #include <linux/compiler.h>
20 #include <linux/ccp.h>
24 static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
26 struct ccp_device *ccp;
29 /* First look at the map for the queue */
30 if (cmd_q->lsb >= 0) {
31 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
34 if (start < LSB_SIZE) {
35 bitmap_set(cmd_q->lsbmap, start, count);
36 return start + cmd_q->lsb * LSB_SIZE;
40 /* No joy; try to get an entry from the shared blocks */
43 mutex_lock(&ccp->sb_mutex);
45 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
46 MAX_LSB_CNT * LSB_SIZE,
49 if (start <= MAX_LSB_CNT * LSB_SIZE) {
50 bitmap_set(ccp->lsbmap, start, count);
52 mutex_unlock(&ccp->sb_mutex);
53 return start * LSB_ITEM_SIZE;
58 mutex_unlock(&ccp->sb_mutex);
60 /* Wait for KSB entries to become available */
61 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
66 static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
69 int lsbno = start / LSB_SIZE;
74 if (cmd_q->lsb == lsbno) {
75 /* An entry from the private LSB */
76 bitmap_clear(cmd_q->lsbmap, start % LSB_SIZE, count);
78 /* From the shared LSBs */
79 struct ccp_device *ccp = cmd_q->ccp;
81 mutex_lock(&ccp->sb_mutex);
82 bitmap_clear(ccp->lsbmap, start, count);
84 mutex_unlock(&ccp->sb_mutex);
85 wake_up_interruptible_all(&ccp->sb_queue);
89 /* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
129 #define CCP_AES_SIZE(p) ((p)->aes.size)
130 #define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)
131 #define CCP_AES_MODE(p) ((p)->aes.mode)
132 #define CCP_AES_TYPE(p) ((p)->aes.type)
133 #define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
134 #define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
135 #define CCP_SHA_TYPE(p) ((p)->sha.type)
136 #define CCP_RSA_SIZE(p) ((p)->rsa.size)
137 #define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)
138 #define CCP_PT_BITWISE(p) ((p)->pt.bitwise)
139 #define CCP_ECC_MODE(p) ((p)->ecc.mode)
140 #define CCP_ECC_AFFINE(p) ((p)->ecc.one)
143 #define CCP5_CMD_DW0(p) ((p)->dw0)
144 #define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)
145 #define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)
146 #define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)
147 #define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)
148 #define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)
149 #define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)
150 #define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)
153 #define CCP5_CMD_DW1(p) ((p)->length)
154 #define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))
157 #define CCP5_CMD_DW2(p) ((p)->src_lo)
158 #define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))
161 #define CCP5_CMD_DW3(p) ((p)->dw3)
162 #define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
163 #define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)
164 #define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
165 #define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)
168 #define CCP5_CMD_DW4(p) ((p)->dw4)
169 #define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)
170 #define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
171 #define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))
172 #define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
173 #define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
174 #define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
175 #define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
178 #define CCP5_CMD_DW6(p) ((p)->key_lo)
179 #define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))
180 #define CCP5_CMD_DW7(p) ((p)->dw7)
181 #define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)
182 #define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
184 static inline u32 low_address(unsigned long addr)
186 return (u64)addr & 0x0ffffffff;
189 static inline u32 high_address(unsigned long addr)
191 return ((u64)addr >> 32) & 0x00000ffff;
194 static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
196 unsigned int head_idx, n;
197 u32 head_lo, queue_start;
199 queue_start = low_address(cmd_q->qdma_tail);
200 head_lo = ioread32(cmd_q->reg_head_lo);
201 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
203 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
205 return n % COMMANDS_PER_QUEUE; /* Always one unused spot */
208 static int ccp5_do_cmd(struct ccp5_desc *desc,
209 struct ccp_cmd_queue *cmd_q)
217 if (CCP5_CMD_SOC(desc)) {
218 CCP5_CMD_IOC(desc) = 1;
219 CCP5_CMD_SOC(desc) = 0;
221 mutex_lock(&cmd_q->q_mutex);
223 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx];
224 dP = (__le32 *) desc;
225 for (i = 0; i < 8; i++)
226 mP[i] = cpu_to_le32(dP[i]); /* handle endianness */
228 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
230 /* The data used by this command must be flushed to memory */
233 /* Write the new tail address back to the queue register */
234 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
235 iowrite32(tail, cmd_q->reg_tail_lo);
237 /* Turn the queue back on using our cached control register */
238 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
239 mutex_unlock(&cmd_q->q_mutex);
241 if (CCP5_CMD_IOC(desc)) {
242 /* Wait for the job to complete */
243 ret = wait_event_interruptible(cmd_q->int_queue,
245 if (ret || cmd_q->cmd_error) {
246 if (cmd_q->cmd_error)
247 ccp_log_error(cmd_q->ccp,
249 /* A version 5 device doesn't use Job IDs... */
259 static int ccp5_perform_aes(struct ccp_op *op)
261 struct ccp5_desc desc;
262 union ccp_function function;
263 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
265 /* Zero out all the fields of the command desc */
266 memset(&desc, 0, Q_DESC_SIZE);
268 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
270 CCP5_CMD_SOC(&desc) = op->soc;
271 CCP5_CMD_IOC(&desc) = 1;
272 CCP5_CMD_INIT(&desc) = op->init;
273 CCP5_CMD_EOM(&desc) = op->eom;
274 CCP5_CMD_PROT(&desc) = 0;
277 CCP_AES_ENCRYPT(&function) = op->u.aes.action;
278 CCP_AES_MODE(&function) = op->u.aes.mode;
279 CCP_AES_TYPE(&function) = op->u.aes.type;
280 if (op->u.aes.mode == CCP_AES_MODE_CFB)
281 CCP_AES_SIZE(&function) = 0x7f;
283 CCP5_CMD_FUNCTION(&desc) = function.raw;
285 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
287 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
288 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
289 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
291 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
292 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
293 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
295 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
296 CCP5_CMD_KEY_HI(&desc) = 0;
297 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
298 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
300 return ccp5_do_cmd(&desc, op->cmd_q);
303 static int ccp5_perform_xts_aes(struct ccp_op *op)
305 struct ccp5_desc desc;
306 union ccp_function function;
307 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
309 /* Zero out all the fields of the command desc */
310 memset(&desc, 0, Q_DESC_SIZE);
312 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
314 CCP5_CMD_SOC(&desc) = op->soc;
315 CCP5_CMD_IOC(&desc) = 1;
316 CCP5_CMD_INIT(&desc) = op->init;
317 CCP5_CMD_EOM(&desc) = op->eom;
318 CCP5_CMD_PROT(&desc) = 0;
321 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
322 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
323 CCP5_CMD_FUNCTION(&desc) = function.raw;
325 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
327 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
328 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
329 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
331 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
332 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
333 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
335 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
336 CCP5_CMD_KEY_HI(&desc) = 0;
337 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
338 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
340 return ccp5_do_cmd(&desc, op->cmd_q);
343 static int ccp5_perform_sha(struct ccp_op *op)
345 struct ccp5_desc desc;
346 union ccp_function function;
348 /* Zero out all the fields of the command desc */
349 memset(&desc, 0, Q_DESC_SIZE);
351 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
353 CCP5_CMD_SOC(&desc) = op->soc;
354 CCP5_CMD_IOC(&desc) = 1;
355 CCP5_CMD_INIT(&desc) = 1;
356 CCP5_CMD_EOM(&desc) = op->eom;
357 CCP5_CMD_PROT(&desc) = 0;
360 CCP_SHA_TYPE(&function) = op->u.sha.type;
361 CCP5_CMD_FUNCTION(&desc) = function.raw;
363 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
365 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
366 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
367 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
369 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
372 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
373 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
375 CCP5_CMD_SHA_LO(&desc) = 0;
376 CCP5_CMD_SHA_HI(&desc) = 0;
379 return ccp5_do_cmd(&desc, op->cmd_q);
382 static int ccp5_perform_rsa(struct ccp_op *op)
384 struct ccp5_desc desc;
385 union ccp_function function;
387 /* Zero out all the fields of the command desc */
388 memset(&desc, 0, Q_DESC_SIZE);
390 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
392 CCP5_CMD_SOC(&desc) = op->soc;
393 CCP5_CMD_IOC(&desc) = 1;
394 CCP5_CMD_INIT(&desc) = 0;
395 CCP5_CMD_EOM(&desc) = 1;
396 CCP5_CMD_PROT(&desc) = 0;
399 CCP_RSA_SIZE(&function) = op->u.rsa.mod_size;
400 CCP5_CMD_FUNCTION(&desc) = function.raw;
402 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
404 /* Source is from external memory */
405 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
406 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
407 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
409 /* Destination is in external memory */
410 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
411 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
412 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
414 /* Key (Exponent) is in external memory */
415 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
416 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
417 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
419 return ccp5_do_cmd(&desc, op->cmd_q);
422 static int ccp5_perform_passthru(struct ccp_op *op)
424 struct ccp5_desc desc;
425 union ccp_function function;
426 struct ccp_dma_info *saddr = &op->src.u.dma;
427 struct ccp_dma_info *daddr = &op->dst.u.dma;
429 memset(&desc, 0, Q_DESC_SIZE);
431 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
433 CCP5_CMD_SOC(&desc) = 0;
434 CCP5_CMD_IOC(&desc) = 1;
435 CCP5_CMD_INIT(&desc) = 0;
436 CCP5_CMD_EOM(&desc) = op->eom;
437 CCP5_CMD_PROT(&desc) = 0;
440 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
441 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
442 CCP5_CMD_FUNCTION(&desc) = function.raw;
444 /* Length of source data is always 256 bytes */
445 if (op->src.type == CCP_MEMTYPE_SYSTEM)
446 CCP5_CMD_LEN(&desc) = saddr->length;
448 CCP5_CMD_LEN(&desc) = daddr->length;
450 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
451 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
452 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
453 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
455 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
456 CCP5_CMD_LSB_ID(&desc) = op->sb_key;
458 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
460 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
461 CCP5_CMD_SRC_HI(&desc) = 0;
462 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
465 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
466 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
467 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
468 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
470 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
472 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
473 CCP5_CMD_DST_HI(&desc) = 0;
474 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
477 return ccp5_do_cmd(&desc, op->cmd_q);
480 static int ccp5_perform_ecc(struct ccp_op *op)
482 struct ccp5_desc desc;
483 union ccp_function function;
485 /* Zero out all the fields of the command desc */
486 memset(&desc, 0, Q_DESC_SIZE);
488 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
490 CCP5_CMD_SOC(&desc) = 0;
491 CCP5_CMD_IOC(&desc) = 1;
492 CCP5_CMD_INIT(&desc) = 0;
493 CCP5_CMD_EOM(&desc) = 1;
494 CCP5_CMD_PROT(&desc) = 0;
497 function.ecc.mode = op->u.ecc.function;
498 CCP5_CMD_FUNCTION(&desc) = function.raw;
500 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
502 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
503 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
504 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
506 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
507 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
508 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
510 return ccp5_do_cmd(&desc, op->cmd_q);
513 static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
515 int q_mask = 1 << cmd_q->id;
519 /* Build a bit mask to know which LSBs this queue has access to.
520 * Don't bother with segment 0 as it has special privileges.
522 for (j = 1; j < MAX_LSB_CNT; j++) {
524 bitmap_set(cmd_q->lsbmask, j, 1);
525 status >>= LSB_REGION_WIDTH;
527 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
528 dev_info(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
531 return queues ? 0 : -EINVAL;
535 static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
536 int lsb_cnt, int n_lsbs,
537 unsigned long *lsb_pub)
539 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
545 * If the count of potential LSBs available to a queue matches the
546 * ordinal given to us in lsb_cnt:
547 * Copy the mask of possible LSBs for this queue into "qlsb";
548 * For each bit in qlsb, see if the corresponding bit in the
549 * aggregation mask is set; if so, we have a match.
550 * If we have a match, clear the bit in the aggregation to
551 * mark it as no longer available.
552 * If there is no match, clear the bit in qlsb and keep looking.
554 for (i = 0; i < ccp->cmd_q_count; i++) {
555 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
557 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
559 if (qlsb_wgt == lsb_cnt) {
560 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
562 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
563 while (bitno < MAX_LSB_CNT) {
564 if (test_bit(bitno, lsb_pub)) {
565 /* We found an available LSB
566 * that this queue can access
569 bitmap_clear(lsb_pub, bitno, 1);
571 "Queue %d gets LSB %d\n",
575 bitmap_clear(qlsb, bitno, 1);
576 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
578 if (bitno >= MAX_LSB_CNT)
586 /* For each queue, from the most- to least-constrained:
587 * find an LSB that can be assigned to the queue. If there are N queues that
588 * can only use M LSBs, where N > M, fail; otherwise, every queue will get a
589 * dedicated LSB. Remaining LSB regions become a shared resource.
590 * If we have fewer LSBs than queues, all LSB regions become shared resources.
592 static int ccp_assign_lsbs(struct ccp_device *ccp)
594 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
595 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
601 bitmap_zero(lsb_pub, MAX_LSB_CNT);
603 /* Create an aggregate bitmap to get a total count of available LSBs */
604 for (i = 0; i < ccp->cmd_q_count; i++)
606 lsb_pub, ccp->cmd_q[i].lsbmask,
609 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
611 if (n_lsbs >= ccp->cmd_q_count) {
612 /* We have enough LSBS to give every queue a private LSB.
613 * Brute force search to start with the queues that are more
614 * constrained in LSB choice. When an LSB is privately
615 * assigned, it is removed from the public mask.
616 * This is an ugly N squared algorithm with some optimization.
619 n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
621 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
630 /* What's left of the LSBs, according to the public mask, now become
631 * shared. Any zero bits in the lsb_pub mask represent an LSB region
632 * that can't be used as a shared resource, so mark the LSB slots for
635 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
637 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
638 while (bitno < MAX_LSB_CNT) {
639 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
640 bitmap_set(qlsb, bitno, 1);
641 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
647 static int ccp5_init(struct ccp_device *ccp)
649 struct device *dev = ccp->dev;
650 struct ccp_cmd_queue *cmd_q;
651 struct dma_pool *dma_pool;
652 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
653 unsigned int qmr, qim, i;
655 u32 status_lo, status_hi;
658 /* Find available queues */
660 qmr = ioread32(ccp->io_regs + Q_MASK_REG);
661 for (i = 0; i < MAX_HW_QUEUES; i++) {
663 if (!(qmr & (1 << i)))
666 /* Allocate a dma pool for this queue */
667 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
669 dma_pool = dma_pool_create(dma_pool_name, dev,
670 CCP_DMAPOOL_MAX_SIZE,
671 CCP_DMAPOOL_ALIGN, 0);
673 dev_err(dev, "unable to allocate dma pool\n");
677 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
682 cmd_q->dma_pool = dma_pool;
683 mutex_init(&cmd_q->q_mutex);
685 /* Page alignment satisfies our needs for N <= 128 */
686 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
687 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
688 cmd_q->qbase = dma_zalloc_coherent(dev, cmd_q->qsize,
692 dev_err(dev, "unable to allocate command queue\n");
698 /* Preset some register values and masks that are queue
701 cmd_q->reg_control = ccp->io_regs +
702 CMD5_Q_STATUS_INCR * (i + 1);
703 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
704 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
705 cmd_q->reg_int_enable = cmd_q->reg_control +
706 CMD5_Q_INT_ENABLE_BASE;
707 cmd_q->reg_interrupt_status = cmd_q->reg_control +
708 CMD5_Q_INTERRUPT_STATUS_BASE;
709 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
710 cmd_q->reg_int_status = cmd_q->reg_control +
711 CMD5_Q_INT_STATUS_BASE;
712 cmd_q->reg_dma_status = cmd_q->reg_control +
713 CMD5_Q_DMA_STATUS_BASE;
714 cmd_q->reg_dma_read_status = cmd_q->reg_control +
715 CMD5_Q_DMA_READ_STATUS_BASE;
716 cmd_q->reg_dma_write_status = cmd_q->reg_control +
717 CMD5_Q_DMA_WRITE_STATUS_BASE;
719 init_waitqueue_head(&cmd_q->int_queue);
721 dev_dbg(dev, "queue #%u available\n", i);
723 if (ccp->cmd_q_count == 0) {
724 dev_notice(dev, "no command queues available\n");
728 dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count);
730 /* Turn off the queues and disable interrupts until ready */
731 for (i = 0; i < ccp->cmd_q_count; i++) {
732 cmd_q = &ccp->cmd_q[i];
734 cmd_q->qcontrol = 0; /* Start with nothing */
735 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
737 /* Disable the interrupts */
738 iowrite32(0x00, cmd_q->reg_int_enable);
739 ioread32(cmd_q->reg_int_status);
740 ioread32(cmd_q->reg_status);
742 /* Clear the interrupts */
743 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
746 dev_dbg(dev, "Requesting an IRQ...\n");
748 ret = ccp->get_irq(ccp);
750 dev_err(dev, "unable to allocate an IRQ\n");
754 /* Initialize the queue used to suspend */
755 init_waitqueue_head(&ccp->suspend_queue);
757 dev_dbg(dev, "Loading LSB map...\n");
758 /* Copy the private LSB mask to the public registers */
759 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
760 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
761 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
762 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
763 status = ((u64)status_hi<<30) | (u64)status_lo;
765 dev_dbg(dev, "Configuring virtual queues...\n");
766 /* Configure size of each virtual queue accessible to host */
767 for (i = 0; i < ccp->cmd_q_count; i++) {
771 cmd_q = &ccp->cmd_q[i];
773 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
774 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
776 cmd_q->qdma_tail = cmd_q->qbase_dma;
777 dma_addr_lo = low_address(cmd_q->qdma_tail);
778 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
779 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
781 dma_addr_hi = high_address(cmd_q->qdma_tail);
782 cmd_q->qcontrol |= (dma_addr_hi << 16);
783 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
785 /* Find the LSB regions accessible to the queue */
786 ccp_find_lsb_regions(cmd_q, status);
787 cmd_q->lsb = -1; /* Unassigned value */
790 dev_dbg(dev, "Assigning LSBs...\n");
791 ret = ccp_assign_lsbs(ccp);
793 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
797 /* Optimization: pre-allocate LSB slots for each queue */
798 for (i = 0; i < ccp->cmd_q_count; i++) {
799 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
800 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
803 dev_dbg(dev, "Starting threads...\n");
804 /* Create a kthread for each queue */
805 for (i = 0; i < ccp->cmd_q_count; i++) {
806 struct task_struct *kthread;
808 cmd_q = &ccp->cmd_q[i];
810 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
811 "%s-q%u", ccp->name, cmd_q->id);
812 if (IS_ERR(kthread)) {
813 dev_err(dev, "error creating queue thread (%ld)\n",
815 ret = PTR_ERR(kthread);
819 cmd_q->kthread = kthread;
820 wake_up_process(kthread);
823 dev_dbg(dev, "Enabling interrupts...\n");
824 /* Enable interrupts */
825 for (i = 0; i < ccp->cmd_q_count; i++) {
826 cmd_q = &ccp->cmd_q[i];
827 iowrite32(ALL_INTERRUPTS, cmd_q->reg_int_enable);
830 dev_dbg(dev, "Registering device...\n");
831 /* Put this on the unit list to make it available */
834 ret = ccp_register_rng(ccp);
838 /* Register the DMA engine support */
839 ret = ccp_dmaengine_register(ccp);
846 ccp_unregister_rng(ccp);
849 for (i = 0; i < ccp->cmd_q_count; i++)
850 if (ccp->cmd_q[i].kthread)
851 kthread_stop(ccp->cmd_q[i].kthread);
857 for (i = 0; i < ccp->cmd_q_count; i++)
858 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
863 static void ccp5_destroy(struct ccp_device *ccp)
865 struct device *dev = ccp->dev;
866 struct ccp_cmd_queue *cmd_q;
870 /* Unregister the DMA engine */
871 ccp_dmaengine_unregister(ccp);
873 /* Unregister the RNG */
874 ccp_unregister_rng(ccp);
876 /* Remove this device from the list of available units first */
879 /* Disable and clear interrupts */
880 for (i = 0; i < ccp->cmd_q_count; i++) {
881 cmd_q = &ccp->cmd_q[i];
883 /* Turn off the run bit */
884 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
886 /* Disable the interrupts */
887 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
889 /* Clear the interrupt status */
890 iowrite32(0x00, cmd_q->reg_int_enable);
891 ioread32(cmd_q->reg_int_status);
892 ioread32(cmd_q->reg_status);
895 /* Stop the queue kthreads */
896 for (i = 0; i < ccp->cmd_q_count; i++)
897 if (ccp->cmd_q[i].kthread)
898 kthread_stop(ccp->cmd_q[i].kthread);
902 for (i = 0; i < ccp->cmd_q_count; i++) {
903 cmd_q = &ccp->cmd_q[i];
904 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
908 /* Flush the cmd and backlog queue */
909 while (!list_empty(&ccp->cmd)) {
910 /* Invoke the callback directly with an error code */
911 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
912 list_del(&cmd->entry);
913 cmd->callback(cmd->data, -ENODEV);
915 while (!list_empty(&ccp->backlog)) {
916 /* Invoke the callback directly with an error code */
917 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
918 list_del(&cmd->entry);
919 cmd->callback(cmd->data, -ENODEV);
923 static irqreturn_t ccp5_irq_handler(int irq, void *data)
925 struct device *dev = data;
926 struct ccp_device *ccp = dev_get_drvdata(dev);
930 for (i = 0; i < ccp->cmd_q_count; i++) {
931 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
933 status = ioread32(cmd_q->reg_interrupt_status);
936 cmd_q->int_status = status;
937 cmd_q->q_status = ioread32(cmd_q->reg_status);
938 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
940 /* On error, only save the first error value */
941 if ((status & INT_ERROR) && !cmd_q->cmd_error)
942 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
946 /* Acknowledge the interrupt and wake the kthread */
947 iowrite32(ALL_INTERRUPTS, cmd_q->reg_interrupt_status);
948 wake_up_interruptible(&cmd_q->int_queue);
955 static void ccp5_config(struct ccp_device *ccp)
958 iowrite32(0x00001249, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
961 static void ccp5other_config(struct ccp_device *ccp)
966 /* We own all of the queues on the NTB CCP */
968 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
969 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
970 for (i = 0; i < 12; i++) {
971 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
972 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
975 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
976 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
977 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
979 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
980 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
982 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
987 /* Version 5 adds some function, but is essentially the same as v5 */
988 static const struct ccp_actions ccp5_actions = {
989 .aes = ccp5_perform_aes,
990 .xts_aes = ccp5_perform_xts_aes,
991 .sha = ccp5_perform_sha,
992 .rsa = ccp5_perform_rsa,
993 .passthru = ccp5_perform_passthru,
994 .ecc = ccp5_perform_ecc,
995 .sballoc = ccp_lsb_alloc,
996 .sbfree = ccp_lsb_free,
998 .destroy = ccp5_destroy,
999 .get_free_slots = ccp5_get_free_slots,
1000 .irqhandler = ccp5_irq_handler,
1003 const struct ccp_vdata ccpv5a = {
1004 .version = CCP_VERSION(5, 0),
1005 .setup = ccp5_config,
1006 .perform = &ccp5_actions,
1011 const struct ccp_vdata ccpv5b = {
1012 .version = CCP_VERSION(5, 0),
1013 .setup = ccp5other_config,
1014 .perform = &ccp5_actions,