drivers/crypto/ccp/ccp-dev.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  * Author: Gary R Hook <gary.hook@amd.com>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2 as
  11  * published by the Free Software Foundation.
  12  */
  13
  14 #include <linux/kernel.h>
  15 #include <linux/kthread.h>
  16 #include <linux/sched.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/spinlock.h>
  19 #include <linux/spinlock_types.h>
  20 #include <linux/types.h>
  21 #include <linux/mutex.h>
  22 #include <linux/delay.h>
  23 #include <linux/hw_random.h>
  24 #include <linux/cpu.h>
  25 #ifdef CONFIG_X86
  26 #include <asm/cpu_device_id.h>
  27 #endif
  28 #include <linux/ccp.h>
  29
  30 #include "ccp-dev.h"
  31
  32 struct ccp_tasklet_data {
  33         struct completion completion;
  34         struct ccp_cmd *cmd;
  35 };
  36
  37 /* Human-readable error strings */
  38 #define CCP_MAX_ERROR_CODE      64
  39 static char *ccp_error_codes[] = {
  40         "",
  41         "ILLEGAL_ENGINE",
  42         "ILLEGAL_KEY_ID",
  43         "ILLEGAL_FUNCTION_TYPE",
  44         "ILLEGAL_FUNCTION_MODE",
  45         "ILLEGAL_FUNCTION_ENCRYPT",
  46         "ILLEGAL_FUNCTION_SIZE",
  47         "Zlib_MISSING_INIT_EOM",
  48         "ILLEGAL_FUNCTION_RSVD",
  49         "ILLEGAL_BUFFER_LENGTH",
  50         "VLSB_FAULT",
  51         "ILLEGAL_MEM_ADDR",
  52         "ILLEGAL_MEM_SEL",
  53         "ILLEGAL_CONTEXT_ID",
  54         "ILLEGAL_KEY_ADDR",
  55         "0xF Reserved",
  56         "Zlib_ILLEGAL_MULTI_QUEUE",
  57         "Zlib_ILLEGAL_JOBID_CHANGE",
  58         "CMD_TIMEOUT",
  59         "IDMA0_AXI_SLVERR",
  60         "IDMA0_AXI_DECERR",
  61         "0x15 Reserved",
  62         "IDMA1_AXI_SLAVE_FAULT",
  63         "IDMA1_AIXI_DECERR",
  64         "0x18 Reserved",
  65         "ZLIBVHB_AXI_SLVERR",
  66         "ZLIBVHB_AXI_DECERR",
  67         "0x1B Reserved",
  68         "ZLIB_UNEXPECTED_EOM",
  69         "ZLIB_EXTRA_DATA",
  70         "ZLIB_BTYPE",
  71         "ZLIB_UNDEFINED_SYMBOL",
  72         "ZLIB_UNDEFINED_DISTANCE_S",
  73         "ZLIB_CODE_LENGTH_SYMBOL",
  74         "ZLIB _VHB_ILLEGAL_FETCH",
  75         "ZLIB_UNCOMPRESSED_LEN",
  76         "ZLIB_LIMIT_REACHED",
  77         "ZLIB_CHECKSUM_MISMATCH0",
  78         "ODMA0_AXI_SLVERR",
  79         "ODMA0_AXI_DECERR",
  80         "0x28 Reserved",
  81         "ODMA1_AXI_SLVERR",
  82         "ODMA1_AXI_DECERR",
  83 };
  84
  85 void ccp_log_error(struct ccp_device *d, unsigned int e)
  86 {
  87         if (WARN_ON(e >= CCP_MAX_ERROR_CODE))
  88                 return;
  89
  90         if (e < ARRAY_SIZE(ccp_error_codes))
  91                 dev_err(d->dev, "CCP error %d: %s\n", e, ccp_error_codes[e]);
  92         else
  93                 dev_err(d->dev, "CCP error %d: Unknown Error\n", e);
  94 }
  95
  96 /* List of CCPs, CCP count, read-write access lock, and access functions
  97  *
  98  * Lock structure: get ccp_unit_lock for reading whenever we need to
  99  * examine the CCP list. While holding it for reading we can acquire
 100  * the RR lock to update the round-robin next-CCP pointer. The unit lock
 101  * must be acquired before the RR lock.
 102  *
 103  * If the unit-lock is acquired for writing, we have total control over
 104  * the list, so there's no value in getting the RR lock.
 105  */
 106 static DEFINE_RWLOCK(ccp_unit_lock);
 107 static LIST_HEAD(ccp_units);
 108
 109 /* Round-robin counter */
 110 static DEFINE_SPINLOCK(ccp_rr_lock);
 111 static struct ccp_device *ccp_rr;
 112
 113 /**
 114  * ccp_add_device - add a CCP device to the list
 115  *
 116  * @ccp: ccp_device struct pointer
 117  *
 118  * Put this CCP on the unit list, which makes it available
 119  * for use.
 120  *
 121  * Returns zero if a CCP device is present, -ENODEV otherwise.
 122  */
 123 void ccp_add_device(struct ccp_device *ccp)
 124 {
 125         unsigned long flags;
 126
 127         write_lock_irqsave(&ccp_unit_lock, flags);
 128         list_add_tail(&ccp->entry, &ccp_units);
 129         if (!ccp_rr)
 130                 /* We already have the list lock (we're first) so this
 131                  * pointer can't change on us. Set its initial value.
 132                  */
 133                 ccp_rr = ccp;
 134         write_unlock_irqrestore(&ccp_unit_lock, flags);
 135 }
 136
 137 /**
 138  * ccp_del_device - remove a CCP device from the list
 139  *
 140  * @ccp: ccp_device struct pointer
 141  *
 142  * Remove this unit from the list of devices. If the next device
 143  * up for use is this one, adjust the pointer. If this is the last
 144  * device, NULL the pointer.
 145  */
 146 void ccp_del_device(struct ccp_device *ccp)
 147 {
 148         unsigned long flags;
 149
 150         write_lock_irqsave(&ccp_unit_lock, flags);
 151         if (ccp_rr == ccp) {
 152                 /* ccp_unit_lock is read/write; any read access
 153                  * will be suspended while we make changes to the
 154                  * list and RR pointer.
 155                  */
 156                 if (list_is_last(&ccp_rr->entry, &ccp_units))
 157                         ccp_rr = list_first_entry(&ccp_units, struct ccp_device,
 158                                                   entry);
 159                 else
 160                         ccp_rr = list_next_entry(ccp_rr, entry);
 161         }
 162         list_del(&ccp->entry);
 163         if (list_empty(&ccp_units))
 164                 ccp_rr = NULL;
 165         write_unlock_irqrestore(&ccp_unit_lock, flags);
 166 }
 167
 168
 169
 170 int ccp_register_rng(struct ccp_device *ccp)
 171 {
 172         int ret = 0;
 173
 174         dev_dbg(ccp->dev, "Registering RNG...\n");
 175         /* Register an RNG */
 176         ccp->hwrng.name = ccp->rngname;
 177         ccp->hwrng.read = ccp_trng_read;
 178         ret = hwrng_register(&ccp->hwrng);
 179         if (ret)
 180                 dev_err(ccp->dev, "error registering hwrng (%d)\n", ret);
 181
 182         return ret;
 183 }
 184
 185 void ccp_unregister_rng(struct ccp_device *ccp)
 186 {
 187         if (ccp->hwrng.name)
 188                 hwrng_unregister(&ccp->hwrng);
 189 }
 190
 191 static struct ccp_device *ccp_get_device(void)
 192 {
 193         unsigned long flags;
 194         struct ccp_device *dp = NULL;
 195
 196         /* We round-robin through the unit list.
 197          * The (ccp_rr) pointer refers to the next unit to use.
 198          */
 199         read_lock_irqsave(&ccp_unit_lock, flags);
 200         if (!list_empty(&ccp_units)) {
 201                 spin_lock(&ccp_rr_lock);
 202                 dp = ccp_rr;
 203                 if (list_is_last(&ccp_rr->entry, &ccp_units))
 204                         ccp_rr = list_first_entry(&ccp_units, struct ccp_device,
 205                                                   entry);
 206                 else
 207                         ccp_rr = list_next_entry(ccp_rr, entry);
 208                 spin_unlock(&ccp_rr_lock);
 209         }
 210         read_unlock_irqrestore(&ccp_unit_lock, flags);
 211
 212         return dp;
 213 }
 214
 215 /**
 216  * ccp_present - check if a CCP device is present
 217  *
 218  * Returns zero if a CCP device is present, -ENODEV otherwise.
 219  */
 220 int ccp_present(void)
 221 {
 222         unsigned long flags;
 223         int ret;
 224
 225         read_lock_irqsave(&ccp_unit_lock, flags);
 226         ret = list_empty(&ccp_units);
 227         read_unlock_irqrestore(&ccp_unit_lock, flags);
 228
 229         return ret ? -ENODEV : 0;
 230 }
 231 EXPORT_SYMBOL_GPL(ccp_present);
 232
 233 /**
 234  * ccp_version - get the version of the CCP device
 235  *
 236  * Returns the version from the first unit on the list;
 237  * otherwise a zero if no CCP device is present
 238  */
 239 unsigned int ccp_version(void)
 240 {
 241         struct ccp_device *dp;
 242         unsigned long flags;
 243         int ret = 0;
 244
 245         read_lock_irqsave(&ccp_unit_lock, flags);
 246         if (!list_empty(&ccp_units)) {
 247                 dp = list_first_entry(&ccp_units, struct ccp_device, entry);
 248                 ret = dp->vdata->version;
 249         }
 250         read_unlock_irqrestore(&ccp_unit_lock, flags);
 251
 252         return ret;
 253 }
 254 EXPORT_SYMBOL_GPL(ccp_version);
 255
 256 /**
 257  * ccp_enqueue_cmd - queue an operation for processing by the CCP
 258  *
 259  * @cmd: ccp_cmd struct to be processed
 260  *
 261  * Queue a cmd to be processed by the CCP. If queueing the cmd
 262  * would exceed the defined length of the cmd queue the cmd will
 263  * only be queued if the CCP_CMD_MAY_BACKLOG flag is set and will
 264  * result in a return code of -EBUSY.
 265  *
 266  * The callback routine specified in the ccp_cmd struct will be
 267  * called to notify the caller of completion (if the cmd was not
 268  * backlogged) or advancement out of the backlog. If the cmd has
 269  * advanced out of the backlog the "err" value of the callback
 270  * will be -EINPROGRESS. Any other "err" value during callback is
 271  * the result of the operation.
 272  *
 273  * The cmd has been successfully queued if:
 274  *   the return code is -EINPROGRESS or
 275  *   the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set
 276  */
 277 int ccp_enqueue_cmd(struct ccp_cmd *cmd)
 278 {
 279         struct ccp_device *ccp;
 280         unsigned long flags;
 281         unsigned int i;
 282         int ret;
 283
 284         /* Some commands might need to be sent to a specific device */
 285         ccp = cmd->ccp ? cmd->ccp : ccp_get_device();
 286
 287         if (!ccp)
 288                 return -ENODEV;
 289
 290         /* Caller must supply a callback routine */
 291         if (!cmd->callback)
 292                 return -EINVAL;
 293
 294         cmd->ccp = ccp;
 295
 296         spin_lock_irqsave(&ccp->cmd_lock, flags);
 297
 298         i = ccp->cmd_q_count;
 299
 300         if (ccp->cmd_count >= MAX_CMD_QLEN) {
 301                 if (cmd->flags & CCP_CMD_MAY_BACKLOG) {
 302                         ret = -EBUSY;
 303                         list_add_tail(&cmd->entry, &ccp->backlog);
 304                 } else {
 305                         ret = -ENOSPC;
 306                 }
 307         } else {
 308                 ret = -EINPROGRESS;
 309                 ccp->cmd_count++;
 310                 list_add_tail(&cmd->entry, &ccp->cmd);
 311
 312                 /* Find an idle queue */
 313                 if (!ccp->suspending) {
 314                         for (i = 0; i < ccp->cmd_q_count; i++) {
 315                                 if (ccp->cmd_q[i].active)
 316                                         continue;
 317
 318                                 break;
 319                         }
 320                 }
 321         }
 322
 323         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 324
 325         /* If we found an idle queue, wake it up */
 326         if (i < ccp->cmd_q_count)
 327                 wake_up_process(ccp->cmd_q[i].kthread);
 328
 329         return ret;
 330 }
 331 EXPORT_SYMBOL_GPL(ccp_enqueue_cmd);
 332
 333 static void ccp_do_cmd_backlog(struct work_struct *work)
 334 {
 335         struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work);
 336         struct ccp_device *ccp = cmd->ccp;
 337         unsigned long flags;
 338         unsigned int i;
 339
 340         cmd->callback(cmd->data, -EINPROGRESS);
 341
 342         spin_lock_irqsave(&ccp->cmd_lock, flags);
 343
 344         ccp->cmd_count++;
 345         list_add_tail(&cmd->entry, &ccp->cmd);
 346
 347         /* Find an idle queue */
 348         for (i = 0; i < ccp->cmd_q_count; i++) {
 349                 if (ccp->cmd_q[i].active)
 350                         continue;
 351
 352                 break;
 353         }
 354
 355         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 356
 357         /* If we found an idle queue, wake it up */
 358         if (i < ccp->cmd_q_count)
 359                 wake_up_process(ccp->cmd_q[i].kthread);
 360 }
 361
 362 static struct ccp_cmd *ccp_dequeue_cmd(struct ccp_cmd_queue *cmd_q)
 363 {
 364         struct ccp_device *ccp = cmd_q->ccp;
 365         struct ccp_cmd *cmd = NULL;
 366         struct ccp_cmd *backlog = NULL;
 367         unsigned long flags;
 368
 369         spin_lock_irqsave(&ccp->cmd_lock, flags);
 370
 371         cmd_q->active = 0;
 372
 373         if (ccp->suspending) {
 374                 cmd_q->suspended = 1;
 375
 376                 spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 377                 wake_up_interruptible(&ccp->suspend_queue);
 378
 379                 return NULL;
 380         }
 381
 382         if (ccp->cmd_count) {
 383                 cmd_q->active = 1;
 384
 385                 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
 386                 list_del(&cmd->entry);
 387
 388                 ccp->cmd_count--;
 389         }
 390
 391         if (!list_empty(&ccp->backlog)) {
 392                 backlog = list_first_entry(&ccp->backlog, struct ccp_cmd,
 393                                            entry);
 394                 list_del(&backlog->entry);
 395         }
 396
 397         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 398
 399         if (backlog) {
 400                 INIT_WORK(&backlog->work, ccp_do_cmd_backlog);
 401                 schedule_work(&backlog->work);
 402         }
 403
 404         return cmd;
 405 }
 406
 407 static void ccp_do_cmd_complete(unsigned long data)
 408 {
 409         struct ccp_tasklet_data *tdata = (struct ccp_tasklet_data *)data;
 410         struct ccp_cmd *cmd = tdata->cmd;
 411
 412         cmd->callback(cmd->data, cmd->ret);
 413
 414         complete(&tdata->completion);
 415 }
 416
 417 /**
 418  * ccp_cmd_queue_thread - create a kernel thread to manage a CCP queue
 419  *
 420  * @data: thread-specific data
 421  */
 422 int ccp_cmd_queue_thread(void *data)
 423 {
 424         struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data;
 425         struct ccp_cmd *cmd;
 426         struct ccp_tasklet_data tdata;
 427         struct tasklet_struct tasklet;
 428
 429         tasklet_init(&tasklet, ccp_do_cmd_complete, (unsigned long)&tdata);
 430
 431         set_current_state(TASK_INTERRUPTIBLE);
 432         while (!kthread_should_stop()) {
 433                 schedule();
 434
 435                 set_current_state(TASK_INTERRUPTIBLE);
 436
 437                 cmd = ccp_dequeue_cmd(cmd_q);
 438                 if (!cmd)
 439                         continue;
 440
 441                 __set_current_state(TASK_RUNNING);
 442
 443                 /* Execute the command */
 444                 cmd->ret = ccp_run_cmd(cmd_q, cmd);
 445
 446                 /* Schedule the completion callback */
 447                 tdata.cmd = cmd;
 448                 init_completion(&tdata.completion);
 449                 tasklet_schedule(&tasklet);
 450                 wait_for_completion(&tdata.completion);
 451         }
 452
 453         __set_current_state(TASK_RUNNING);
 454
 455         return 0;
 456 }
 457
 458 /**
 459  * ccp_alloc_struct - allocate and initialize the ccp_device struct
 460  *
 461  * @dev: device struct of the CCP
 462  */
 463 struct ccp_device *ccp_alloc_struct(struct sp_device *sp)
 464 {
 465         struct device *dev = sp->dev;
 466         struct ccp_device *ccp;
 467
 468         ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL);
 469         if (!ccp)
 470                 return NULL;
 471         ccp->dev = dev;
 472         ccp->sp = sp;
 473         ccp->axcache = sp->axcache;
 474
 475         INIT_LIST_HEAD(&ccp->cmd);
 476         INIT_LIST_HEAD(&ccp->backlog);
 477
 478         spin_lock_init(&ccp->cmd_lock);
 479         mutex_init(&ccp->req_mutex);
 480         mutex_init(&ccp->sb_mutex);
 481         ccp->sb_count = KSB_COUNT;
 482         ccp->sb_start = 0;
 483
 484         /* Initialize the wait queues */
 485         init_waitqueue_head(&ccp->sb_queue);
 486         init_waitqueue_head(&ccp->suspend_queue);
 487
 488         snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", sp->ord);
 489         snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", sp->ord);
 490
 491         return ccp;
 492 }
 493
 494 int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
 495 {
 496         struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng);
 497         u32 trng_value;
 498         int len = min_t(int, sizeof(trng_value), max);
 499
 500         /* Locking is provided by the caller so we can update device
 501          * hwrng-related fields safely
 502          */
 503         trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG);
 504         if (!trng_value) {
 505                 /* Zero is returned if not data is available or if a
 506                  * bad-entropy error is present. Assume an error if
 507                  * we exceed TRNG_RETRIES reads of zero.
 508                  */
 509                 if (ccp->hwrng_retries++ > TRNG_RETRIES)
 510                         return -EIO;
 511
 512                 return 0;
 513         }
 514
 515         /* Reset the counter and save the rng value */
 516         ccp->hwrng_retries = 0;
 517         memcpy(data, &trng_value, len);
 518
 519         return len;
 520 }
 521
 522 #ifdef CONFIG_PM
 523 bool ccp_queues_suspended(struct ccp_device *ccp)
 524 {
 525         unsigned int suspended = 0;
 526         unsigned long flags;
 527         unsigned int i;
 528
 529         spin_lock_irqsave(&ccp->cmd_lock, flags);
 530
 531         for (i = 0; i < ccp->cmd_q_count; i++)
 532                 if (ccp->cmd_q[i].suspended)
 533                         suspended++;
 534
 535         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 536
 537         return ccp->cmd_q_count == suspended;
 538 }
 539
 540 int ccp_dev_suspend(struct sp_device *sp, pm_message_t state)
 541 {
 542         struct ccp_device *ccp = sp->ccp_data;
 543         unsigned long flags;
 544         unsigned int i;
 545
 546         /* If there's no device there's nothing to do */
 547         if (!ccp)
 548                 return 0;
 549
 550         spin_lock_irqsave(&ccp->cmd_lock, flags);
 551
 552         ccp->suspending = 1;
 553
 554         /* Wake all the queue kthreads to prepare for suspend */
 555         for (i = 0; i < ccp->cmd_q_count; i++)
 556                 wake_up_process(ccp->cmd_q[i].kthread);
 557
 558         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 559
 560         /* Wait for all queue kthreads to say they're done */
 561         while (!ccp_queues_suspended(ccp))
 562                 wait_event_interruptible(ccp->suspend_queue,
 563                                          ccp_queues_suspended(ccp));
 564
 565         return 0;
 566 }
 567
 568 int ccp_dev_resume(struct sp_device *sp)
 569 {
 570         struct ccp_device *ccp = sp->ccp_data;
 571         unsigned long flags;
 572         unsigned int i;
 573
 574         /* If there's no device there's nothing to do */
 575         if (!ccp)
 576                 return 0;
 577
 578         spin_lock_irqsave(&ccp->cmd_lock, flags);
 579
 580         ccp->suspending = 0;
 581
 582         /* Wake up all the kthreads */
 583         for (i = 0; i < ccp->cmd_q_count; i++) {
 584                 ccp->cmd_q[i].suspended = 0;
 585                 wake_up_process(ccp->cmd_q[i].kthread);
 586         }
 587
 588         spin_unlock_irqrestore(&ccp->cmd_lock, flags);
 589
 590         return 0;
 591 }
 592 #endif
 593
 594 int ccp_dev_init(struct sp_device *sp)
 595 {
 596         struct device *dev = sp->dev;
 597         struct ccp_device *ccp;
 598         int ret;
 599
 600         ret = -ENOMEM;
 601         ccp = ccp_alloc_struct(sp);
 602         if (!ccp)
 603                 goto e_err;
 604         sp->ccp_data = ccp;
 605
 606         ccp->vdata = (struct ccp_vdata *)sp->dev_vdata->ccp_vdata;
 607         if (!ccp->vdata || !ccp->vdata->version) {
 608                 ret = -ENODEV;
 609                 dev_err(dev, "missing driver data\n");
 610                 goto e_err;
 611         }
 612
 613         ccp->use_tasklet = sp->use_tasklet;
 614
 615         ccp->io_regs = sp->io_map + ccp->vdata->offset;
 616         if (ccp->vdata->setup)
 617                 ccp->vdata->setup(ccp);
 618
 619         ret = ccp->vdata->perform->init(ccp);
 620         if (ret)
 621                 goto e_err;
 622
 623         dev_notice(dev, "ccp enabled\n");
 624
 625         return 0;
 626
 627 e_err:
 628         sp->ccp_data = NULL;
 629
 630         dev_notice(dev, "ccp initialization failed\n");
 631
 632         return ret;
 633 }
 634
 635 void ccp_dev_destroy(struct sp_device *sp)
 636 {
 637         struct ccp_device *ccp = sp->ccp_data;
 638
 639         if (!ccp)
 640                 return;
 641
 642         ccp->vdata->perform->destroy(ccp);
 643 }