drivers/crypto/img-hash.c

   1 /*
   2  * Copyright (c) 2014 Imagination Technologies
   3  * Authors:  Will Thomas, James Hartley
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as published
   7  * by the Free Software Foundation.
   8  *
   9  *      Interface structure taken from omap-sham driver
  10  */
  11
  12 #include <linux/clk.h>
  13 #include <linux/dmaengine.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/io.h>
  16 #include <linux/kernel.h>
  17 #include <linux/module.h>
  18 #include <linux/of_device.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/scatterlist.h>
  21
  22 #include <crypto/internal/hash.h>
  23 #include <crypto/md5.h>
  24 #include <crypto/sha.h>
  25
  26 #define CR_RESET                        0
  27 #define CR_RESET_SET                    1
  28 #define CR_RESET_UNSET                  0
  29
  30 #define CR_MESSAGE_LENGTH_H             0x4
  31 #define CR_MESSAGE_LENGTH_L             0x8
  32
  33 #define CR_CONTROL                      0xc
  34 #define CR_CONTROL_BYTE_ORDER_3210      0
  35 #define CR_CONTROL_BYTE_ORDER_0123      1
  36 #define CR_CONTROL_BYTE_ORDER_2310      2
  37 #define CR_CONTROL_BYTE_ORDER_1032      3
  38 #define CR_CONTROL_BYTE_ORDER_SHIFT     8
  39 #define CR_CONTROL_ALGO_MD5     0
  40 #define CR_CONTROL_ALGO_SHA1    1
  41 #define CR_CONTROL_ALGO_SHA224  2
  42 #define CR_CONTROL_ALGO_SHA256  3
  43
  44 #define CR_INTSTAT                      0x10
  45 #define CR_INTENAB                      0x14
  46 #define CR_INTCLEAR                     0x18
  47 #define CR_INT_RESULTS_AVAILABLE        BIT(0)
  48 #define CR_INT_NEW_RESULTS_SET          BIT(1)
  49 #define CR_INT_RESULT_READ_ERR          BIT(2)
  50 #define CR_INT_MESSAGE_WRITE_ERROR      BIT(3)
  51 #define CR_INT_STATUS                   BIT(8)
  52
  53 #define CR_RESULT_QUEUE         0x1c
  54 #define CR_RSD0                         0x40
  55 #define CR_CORE_REV                     0x50
  56 #define CR_CORE_DES1            0x60
  57 #define CR_CORE_DES2            0x70
  58
  59 #define DRIVER_FLAGS_BUSY               BIT(0)
  60 #define DRIVER_FLAGS_FINAL              BIT(1)
  61 #define DRIVER_FLAGS_DMA_ACTIVE         BIT(2)
  62 #define DRIVER_FLAGS_OUTPUT_READY       BIT(3)
  63 #define DRIVER_FLAGS_INIT               BIT(4)
  64 #define DRIVER_FLAGS_CPU                BIT(5)
  65 #define DRIVER_FLAGS_DMA_READY          BIT(6)
  66 #define DRIVER_FLAGS_ERROR              BIT(7)
  67 #define DRIVER_FLAGS_SG                 BIT(8)
  68 #define DRIVER_FLAGS_SHA1               BIT(18)
  69 #define DRIVER_FLAGS_SHA224             BIT(19)
  70 #define DRIVER_FLAGS_SHA256             BIT(20)
  71 #define DRIVER_FLAGS_MD5                BIT(21)
  72
  73 #define IMG_HASH_QUEUE_LENGTH           20
  74 #define IMG_HASH_DMA_BURST              4
  75 #define IMG_HASH_DMA_THRESHOLD          64
  76
  77 #ifdef __LITTLE_ENDIAN
  78 #define IMG_HASH_BYTE_ORDER             CR_CONTROL_BYTE_ORDER_3210
  79 #else
  80 #define IMG_HASH_BYTE_ORDER             CR_CONTROL_BYTE_ORDER_0123
  81 #endif
  82
  83 struct img_hash_dev;
  84
  85 struct img_hash_request_ctx {
  86         struct img_hash_dev     *hdev;
  87         u8 digest[SHA256_DIGEST_SIZE] __aligned(sizeof(u32));
  88         unsigned long           flags;
  89         size_t                  digsize;
  90
  91         dma_addr_t              dma_addr;
  92         size_t                  dma_ct;
  93
  94         /* sg root */
  95         struct scatterlist      *sgfirst;
  96         /* walk state */
  97         struct scatterlist      *sg;
  98         size_t                  nents;
  99         size_t                  offset;
 100         unsigned int            total;
 101         size_t                  sent;
 102
 103         unsigned long           op;
 104
 105         size_t                  bufcnt;
 106         struct ahash_request    fallback_req;
 107
 108         /* Zero length buffer must remain last member of struct */
 109         u8 buffer[0] __aligned(sizeof(u32));
 110 };
 111
 112 struct img_hash_ctx {
 113         struct img_hash_dev     *hdev;
 114         unsigned long           flags;
 115         struct crypto_ahash     *fallback;
 116 };
 117
 118 struct img_hash_dev {
 119         struct list_head        list;
 120         struct device           *dev;
 121         struct clk              *hash_clk;
 122         struct clk              *sys_clk;
 123         void __iomem            *io_base;
 124
 125         phys_addr_t             bus_addr;
 126         void __iomem            *cpu_addr;
 127
 128         spinlock_t              lock;
 129         int                     err;
 130         struct tasklet_struct   done_task;
 131         struct tasklet_struct   dma_task;
 132
 133         unsigned long           flags;
 134         struct crypto_queue     queue;
 135         struct ahash_request    *req;
 136
 137         struct dma_chan         *dma_lch;
 138 };
 139
 140 struct img_hash_drv {
 141         struct list_head dev_list;
 142         spinlock_t lock;
 143 };
 144
 145 static struct img_hash_drv img_hash = {
 146         .dev_list = LIST_HEAD_INIT(img_hash.dev_list),
 147         .lock = __SPIN_LOCK_UNLOCKED(img_hash.lock),
 148 };
 149
 150 static inline u32 img_hash_read(struct img_hash_dev *hdev, u32 offset)
 151 {
 152         return readl_relaxed(hdev->io_base + offset);
 153 }
 154
 155 static inline void img_hash_write(struct img_hash_dev *hdev,
 156                                   u32 offset, u32 value)
 157 {
 158         writel_relaxed(value, hdev->io_base + offset);
 159 }
 160
 161 static inline u32 img_hash_read_result_queue(struct img_hash_dev *hdev)
 162 {
 163         return be32_to_cpu(img_hash_read(hdev, CR_RESULT_QUEUE));
 164 }
 165
 166 static void img_hash_start(struct img_hash_dev *hdev, bool dma)
 167 {
 168         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 169         u32 cr = IMG_HASH_BYTE_ORDER << CR_CONTROL_BYTE_ORDER_SHIFT;
 170
 171         if (ctx->flags & DRIVER_FLAGS_MD5)
 172                 cr |= CR_CONTROL_ALGO_MD5;
 173         else if (ctx->flags & DRIVER_FLAGS_SHA1)
 174                 cr |= CR_CONTROL_ALGO_SHA1;
 175         else if (ctx->flags & DRIVER_FLAGS_SHA224)
 176                 cr |= CR_CONTROL_ALGO_SHA224;
 177         else if (ctx->flags & DRIVER_FLAGS_SHA256)
 178                 cr |= CR_CONTROL_ALGO_SHA256;
 179         dev_dbg(hdev->dev, "Starting hash process\n");
 180         img_hash_write(hdev, CR_CONTROL, cr);
 181
 182         /*
 183          * The hardware block requires two cycles between writing the control
 184          * register and writing the first word of data in non DMA mode, to
 185          * ensure the first data write is not grouped in burst with the control
 186          * register write a read is issued to 'flush' the bus.
 187          */
 188         if (!dma)
 189                 img_hash_read(hdev, CR_CONTROL);
 190 }
 191
 192 static int img_hash_xmit_cpu(struct img_hash_dev *hdev, const u8 *buf,
 193                              size_t length, int final)
 194 {
 195         u32 count, len32;
 196         const u32 *buffer = (const u32 *)buf;
 197
 198         dev_dbg(hdev->dev, "xmit_cpu:  length: %zu bytes\n", length);
 199
 200         if (final)
 201                 hdev->flags |= DRIVER_FLAGS_FINAL;
 202
 203         len32 = DIV_ROUND_UP(length, sizeof(u32));
 204
 205         for (count = 0; count < len32; count++)
 206                 writel_relaxed(buffer[count], hdev->cpu_addr);
 207
 208         return -EINPROGRESS;
 209 }
 210
 211 static void img_hash_dma_callback(void *data)
 212 {
 213         struct img_hash_dev *hdev = (struct img_hash_dev *)data;
 214         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 215
 216         if (ctx->bufcnt) {
 217                 img_hash_xmit_cpu(hdev, ctx->buffer, ctx->bufcnt, 0);
 218                 ctx->bufcnt = 0;
 219         }
 220         if (ctx->sg)
 221                 tasklet_schedule(&hdev->dma_task);
 222 }
 223
 224 static int img_hash_xmit_dma(struct img_hash_dev *hdev, struct scatterlist *sg)
 225 {
 226         struct dma_async_tx_descriptor *desc;
 227         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 228
 229         ctx->dma_ct = dma_map_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
 230         if (ctx->dma_ct == 0) {
 231                 dev_err(hdev->dev, "Invalid DMA sg\n");
 232                 hdev->err = -EINVAL;
 233                 return -EINVAL;
 234         }
 235
 236         desc = dmaengine_prep_slave_sg(hdev->dma_lch,
 237                                        sg,
 238                                        ctx->dma_ct,
 239                                        DMA_MEM_TO_DEV,
 240                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 241         if (!desc) {
 242                 dev_err(hdev->dev, "Null DMA descriptor\n");
 243                 hdev->err = -EINVAL;
 244                 dma_unmap_sg(hdev->dev, sg, 1, DMA_TO_DEVICE);
 245                 return -EINVAL;
 246         }
 247         desc->callback = img_hash_dma_callback;
 248         desc->callback_param = hdev;
 249         dmaengine_submit(desc);
 250         dma_async_issue_pending(hdev->dma_lch);
 251
 252         return 0;
 253 }
 254
 255 static int img_hash_write_via_cpu(struct img_hash_dev *hdev)
 256 {
 257         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 258
 259         ctx->bufcnt = sg_copy_to_buffer(hdev->req->src, sg_nents(ctx->sg),
 260                                         ctx->buffer, hdev->req->nbytes);
 261
 262         ctx->total = hdev->req->nbytes;
 263         ctx->bufcnt = 0;
 264
 265         hdev->flags |= (DRIVER_FLAGS_CPU | DRIVER_FLAGS_FINAL);
 266
 267         img_hash_start(hdev, false);
 268
 269         return img_hash_xmit_cpu(hdev, ctx->buffer, ctx->total, 1);
 270 }
 271
 272 static int img_hash_finish(struct ahash_request *req)
 273 {
 274         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 275
 276         if (!req->result)
 277                 return -EINVAL;
 278
 279         memcpy(req->result, ctx->digest, ctx->digsize);
 280
 281         return 0;
 282 }
 283
 284 static void img_hash_copy_hash(struct ahash_request *req)
 285 {
 286         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 287         u32 *hash = (u32 *)ctx->digest;
 288         int i;
 289
 290         for (i = (ctx->digsize / sizeof(u32)) - 1; i >= 0; i--)
 291                 hash[i] = img_hash_read_result_queue(ctx->hdev);
 292 }
 293
 294 static void img_hash_finish_req(struct ahash_request *req, int err)
 295 {
 296         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 297         struct img_hash_dev *hdev =  ctx->hdev;
 298
 299         if (!err) {
 300                 img_hash_copy_hash(req);
 301                 if (DRIVER_FLAGS_FINAL & hdev->flags)
 302                         err = img_hash_finish(req);
 303         } else {
 304                 dev_warn(hdev->dev, "Hash failed with error %d\n", err);
 305                 ctx->flags |= DRIVER_FLAGS_ERROR;
 306         }
 307
 308         hdev->flags &= ~(DRIVER_FLAGS_DMA_READY | DRIVER_FLAGS_OUTPUT_READY |
 309                 DRIVER_FLAGS_CPU | DRIVER_FLAGS_BUSY | DRIVER_FLAGS_FINAL);
 310
 311         if (req->base.complete)
 312                 req->base.complete(&req->base, err);
 313 }
 314
 315 static int img_hash_write_via_dma(struct img_hash_dev *hdev)
 316 {
 317         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 318
 319         img_hash_start(hdev, true);
 320
 321         dev_dbg(hdev->dev, "xmit dma size: %d\n", ctx->total);
 322
 323         if (!ctx->total)
 324                 hdev->flags |= DRIVER_FLAGS_FINAL;
 325
 326         hdev->flags |= DRIVER_FLAGS_DMA_ACTIVE | DRIVER_FLAGS_FINAL;
 327
 328         tasklet_schedule(&hdev->dma_task);
 329
 330         return -EINPROGRESS;
 331 }
 332
 333 static int img_hash_dma_init(struct img_hash_dev *hdev)
 334 {
 335         struct dma_slave_config dma_conf;
 336         int err = -EINVAL;
 337
 338         hdev->dma_lch = dma_request_slave_channel(hdev->dev, "tx");
 339         if (!hdev->dma_lch) {
 340                 dev_err(hdev->dev, "Couldn't acquire a slave DMA channel.\n");
 341                 return -EBUSY;
 342         }
 343         dma_conf.direction = DMA_MEM_TO_DEV;
 344         dma_conf.dst_addr = hdev->bus_addr;
 345         dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 346         dma_conf.dst_maxburst = IMG_HASH_DMA_BURST;
 347         dma_conf.device_fc = false;
 348
 349         err = dmaengine_slave_config(hdev->dma_lch,  &dma_conf);
 350         if (err) {
 351                 dev_err(hdev->dev, "Couldn't configure DMA slave.\n");
 352                 dma_release_channel(hdev->dma_lch);
 353                 return err;
 354         }
 355
 356         return 0;
 357 }
 358
 359 static void img_hash_dma_task(unsigned long d)
 360 {
 361         struct img_hash_dev *hdev = (struct img_hash_dev *)d;
 362         struct img_hash_request_ctx *ctx;
 363         u8 *addr;
 364         size_t nbytes, bleft, wsend, len, tbc;
 365         struct scatterlist tsg;
 366
 367         if (!hdev->req)
 368                 return;
 369
 370         ctx = ahash_request_ctx(hdev->req);
 371         if (!ctx->sg)
 372                 return;
 373
 374         addr = sg_virt(ctx->sg);
 375         nbytes = ctx->sg->length - ctx->offset;
 376
 377         /*
 378          * The hash accelerator does not support a data valid mask. This means
 379          * that if each dma (i.e. per page) is not a multiple of 4 bytes, the
 380          * padding bytes in the last word written by that dma would erroneously
 381          * be included in the hash. To avoid this we round down the transfer,
 382          * and add the excess to the start of the next dma. It does not matter
 383          * that the final dma may not be a multiple of 4 bytes as the hashing
 384          * block is programmed to accept the correct number of bytes.
 385          */
 386
 387         bleft = nbytes % 4;
 388         wsend = (nbytes / 4);
 389
 390         if (wsend) {
 391                 sg_init_one(&tsg, addr + ctx->offset, wsend * 4);
 392                 if (img_hash_xmit_dma(hdev, &tsg)) {
 393                         dev_err(hdev->dev, "DMA failed, falling back to CPU");
 394                         ctx->flags |= DRIVER_FLAGS_CPU;
 395                         hdev->err = 0;
 396                         img_hash_xmit_cpu(hdev, addr + ctx->offset,
 397                                           wsend * 4, 0);
 398                         ctx->sent += wsend * 4;
 399                         wsend = 0;
 400                 } else {
 401                         ctx->sent += wsend * 4;
 402                 }
 403         }
 404
 405         if (bleft) {
 406                 ctx->bufcnt = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
 407                                                  ctx->buffer, bleft, ctx->sent);
 408                 tbc = 0;
 409                 ctx->sg = sg_next(ctx->sg);
 410                 while (ctx->sg && (ctx->bufcnt < 4)) {
 411                         len = ctx->sg->length;
 412                         if (likely(len > (4 - ctx->bufcnt)))
 413                                 len = 4 - ctx->bufcnt;
 414                         tbc = sg_pcopy_to_buffer(ctx->sgfirst, ctx->nents,
 415                                                  ctx->buffer + ctx->bufcnt, len,
 416                                         ctx->sent + ctx->bufcnt);
 417                         ctx->bufcnt += tbc;
 418                         if (tbc >= ctx->sg->length) {
 419                                 ctx->sg = sg_next(ctx->sg);
 420                                 tbc = 0;
 421                         }
 422                 }
 423
 424                 ctx->sent += ctx->bufcnt;
 425                 ctx->offset = tbc;
 426
 427                 if (!wsend)
 428                         img_hash_dma_callback(hdev);
 429         } else {
 430                 ctx->offset = 0;
 431                 ctx->sg = sg_next(ctx->sg);
 432         }
 433 }
 434
 435 static int img_hash_write_via_dma_stop(struct img_hash_dev *hdev)
 436 {
 437         struct img_hash_request_ctx *ctx = ahash_request_ctx(hdev->req);
 438
 439         if (ctx->flags & DRIVER_FLAGS_SG)
 440                 dma_unmap_sg(hdev->dev, ctx->sg, ctx->dma_ct, DMA_TO_DEVICE);
 441
 442         return 0;
 443 }
 444
 445 static int img_hash_process_data(struct img_hash_dev *hdev)
 446 {
 447         struct ahash_request *req = hdev->req;
 448         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 449         int err = 0;
 450
 451         ctx->bufcnt = 0;
 452
 453         if (req->nbytes >= IMG_HASH_DMA_THRESHOLD) {
 454                 dev_dbg(hdev->dev, "process data request(%d bytes) using DMA\n",
 455                         req->nbytes);
 456                 err = img_hash_write_via_dma(hdev);
 457         } else {
 458                 dev_dbg(hdev->dev, "process data request(%d bytes) using CPU\n",
 459                         req->nbytes);
 460                 err = img_hash_write_via_cpu(hdev);
 461         }
 462         return err;
 463 }
 464
 465 static int img_hash_hw_init(struct img_hash_dev *hdev)
 466 {
 467         unsigned long long nbits;
 468         u32 u, l;
 469
 470         img_hash_write(hdev, CR_RESET, CR_RESET_SET);
 471         img_hash_write(hdev, CR_RESET, CR_RESET_UNSET);
 472         img_hash_write(hdev, CR_INTENAB, CR_INT_NEW_RESULTS_SET);
 473
 474         nbits = (u64)hdev->req->nbytes << 3;
 475         u = nbits >> 32;
 476         l = nbits;
 477         img_hash_write(hdev, CR_MESSAGE_LENGTH_H, u);
 478         img_hash_write(hdev, CR_MESSAGE_LENGTH_L, l);
 479
 480         if (!(DRIVER_FLAGS_INIT & hdev->flags)) {
 481                 hdev->flags |= DRIVER_FLAGS_INIT;
 482                 hdev->err = 0;
 483         }
 484         dev_dbg(hdev->dev, "hw initialized, nbits: %llx\n", nbits);
 485         return 0;
 486 }
 487
 488 static int img_hash_init(struct ahash_request *req)
 489 {
 490         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 491         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 492         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 493
 494         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 495         rctx->fallback_req.base.flags = req->base.flags
 496                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 497
 498         return crypto_ahash_init(&rctx->fallback_req);
 499 }
 500
 501 static int img_hash_handle_queue(struct img_hash_dev *hdev,
 502                                  struct ahash_request *req)
 503 {
 504         struct crypto_async_request *async_req, *backlog;
 505         struct img_hash_request_ctx *ctx;
 506         unsigned long flags;
 507         int err = 0, res = 0;
 508
 509         spin_lock_irqsave(&hdev->lock, flags);
 510
 511         if (req)
 512                 res = ahash_enqueue_request(&hdev->queue, req);
 513
 514         if (DRIVER_FLAGS_BUSY & hdev->flags) {
 515                 spin_unlock_irqrestore(&hdev->lock, flags);
 516                 return res;
 517         }
 518
 519         backlog = crypto_get_backlog(&hdev->queue);
 520         async_req = crypto_dequeue_request(&hdev->queue);
 521         if (async_req)
 522                 hdev->flags |= DRIVER_FLAGS_BUSY;
 523
 524         spin_unlock_irqrestore(&hdev->lock, flags);
 525
 526         if (!async_req)
 527                 return res;
 528
 529         if (backlog)
 530                 backlog->complete(backlog, -EINPROGRESS);
 531
 532         req = ahash_request_cast(async_req);
 533         hdev->req = req;
 534
 535         ctx = ahash_request_ctx(req);
 536
 537         dev_info(hdev->dev, "processing req, op: %lu, bytes: %d\n",
 538                  ctx->op, req->nbytes);
 539
 540         err = img_hash_hw_init(hdev);
 541
 542         if (!err)
 543                 err = img_hash_process_data(hdev);
 544
 545         if (err != -EINPROGRESS) {
 546                 /* done_task will not finish so do it here */
 547                 img_hash_finish_req(req, err);
 548         }
 549         return res;
 550 }
 551
 552 static int img_hash_update(struct ahash_request *req)
 553 {
 554         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 555         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 556         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 557
 558         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 559         rctx->fallback_req.base.flags = req->base.flags
 560                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 561         rctx->fallback_req.nbytes = req->nbytes;
 562         rctx->fallback_req.src = req->src;
 563
 564         return crypto_ahash_update(&rctx->fallback_req);
 565 }
 566
 567 static int img_hash_final(struct ahash_request *req)
 568 {
 569         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 570         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 571         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 572
 573         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 574         rctx->fallback_req.base.flags = req->base.flags
 575                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 576         rctx->fallback_req.result = req->result;
 577
 578         return crypto_ahash_final(&rctx->fallback_req);
 579 }
 580
 581 static int img_hash_finup(struct ahash_request *req)
 582 {
 583         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 584         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 585         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 586
 587         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 588         rctx->fallback_req.base.flags = req->base.flags
 589                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 590         rctx->fallback_req.nbytes = req->nbytes;
 591         rctx->fallback_req.src = req->src;
 592         rctx->fallback_req.result = req->result;
 593
 594         return crypto_ahash_finup(&rctx->fallback_req);
 595 }
 596
 597 static int img_hash_import(struct ahash_request *req, const void *in)
 598 {
 599         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 600         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 601         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 602
 603         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 604         rctx->fallback_req.base.flags = req->base.flags
 605                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 606
 607         return crypto_ahash_import(&rctx->fallback_req, in);
 608 }
 609
 610 static int img_hash_export(struct ahash_request *req, void *out)
 611 {
 612         struct img_hash_request_ctx *rctx = ahash_request_ctx(req);
 613         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 614         struct img_hash_ctx *ctx = crypto_ahash_ctx(tfm);
 615
 616         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback);
 617         rctx->fallback_req.base.flags = req->base.flags
 618                 & CRYPTO_TFM_REQ_MAY_SLEEP;
 619
 620         return crypto_ahash_export(&rctx->fallback_req, out);
 621 }
 622
 623 static int img_hash_digest(struct ahash_request *req)
 624 {
 625         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 626         struct img_hash_ctx *tctx = crypto_ahash_ctx(tfm);
 627         struct img_hash_request_ctx *ctx = ahash_request_ctx(req);
 628         struct img_hash_dev *hdev = NULL;
 629         struct img_hash_dev *tmp;
 630         int err;
 631
 632         spin_lock(&img_hash.lock);
 633         if (!tctx->hdev) {
 634                 list_for_each_entry(tmp, &img_hash.dev_list, list) {
 635                         hdev = tmp;
 636                         break;
 637                 }
 638                 tctx->hdev = hdev;
 639
 640         } else {
 641                 hdev = tctx->hdev;
 642         }
 643
 644         spin_unlock(&img_hash.lock);
 645         ctx->hdev = hdev;
 646         ctx->flags = 0;
 647         ctx->digsize = crypto_ahash_digestsize(tfm);
 648
 649         switch (ctx->digsize) {
 650         case SHA1_DIGEST_SIZE:
 651                 ctx->flags |= DRIVER_FLAGS_SHA1;
 652                 break;
 653         case SHA256_DIGEST_SIZE:
 654                 ctx->flags |= DRIVER_FLAGS_SHA256;
 655                 break;
 656         case SHA224_DIGEST_SIZE:
 657                 ctx->flags |= DRIVER_FLAGS_SHA224;
 658                 break;
 659         case MD5_DIGEST_SIZE:
 660                 ctx->flags |= DRIVER_FLAGS_MD5;
 661                 break;
 662         default:
 663                 return -EINVAL;
 664         }
 665
 666         ctx->bufcnt = 0;
 667         ctx->offset = 0;
 668         ctx->sent = 0;
 669         ctx->total = req->nbytes;
 670         ctx->sg = req->src;
 671         ctx->sgfirst = req->src;
 672         ctx->nents = sg_nents(ctx->sg);
 673
 674         err = img_hash_handle_queue(tctx->hdev, req);
 675
 676         return err;
 677 }
 678
 679 static int img_hash_cra_init(struct crypto_tfm *tfm, const char *alg_name)
 680 {
 681         struct img_hash_ctx *ctx = crypto_tfm_ctx(tfm);
 682         int err = -ENOMEM;
 683
 684         ctx->fallback = crypto_alloc_ahash(alg_name, 0,
 685                                            CRYPTO_ALG_NEED_FALLBACK);
 686         if (IS_ERR(ctx->fallback)) {
 687                 pr_err("img_hash: Could not load fallback driver.\n");
 688                 err = PTR_ERR(ctx->fallback);
 689                 goto err;
 690         }
 691         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 692                                  sizeof(struct img_hash_request_ctx) +
 693                                  crypto_ahash_reqsize(ctx->fallback) +
 694                                  IMG_HASH_DMA_THRESHOLD);
 695
 696         return 0;
 697
 698 err:
 699         return err;
 700 }
 701
 702 static int img_hash_cra_md5_init(struct crypto_tfm *tfm)
 703 {
 704         return img_hash_cra_init(tfm, "md5-generic");
 705 }
 706
 707 static int img_hash_cra_sha1_init(struct crypto_tfm *tfm)
 708 {
 709         return img_hash_cra_init(tfm, "sha1-generic");
 710 }
 711
 712 static int img_hash_cra_sha224_init(struct crypto_tfm *tfm)
 713 {
 714         return img_hash_cra_init(tfm, "sha224-generic");
 715 }
 716
 717 static int img_hash_cra_sha256_init(struct crypto_tfm *tfm)
 718 {
 719         return img_hash_cra_init(tfm, "sha256-generic");
 720 }
 721
 722 static void img_hash_cra_exit(struct crypto_tfm *tfm)
 723 {
 724         struct img_hash_ctx *tctx = crypto_tfm_ctx(tfm);
 725
 726         crypto_free_ahash(tctx->fallback);
 727 }
 728
 729 static irqreturn_t img_irq_handler(int irq, void *dev_id)
 730 {
 731         struct img_hash_dev *hdev = dev_id;
 732         u32 reg;
 733
 734         reg = img_hash_read(hdev, CR_INTSTAT);
 735         img_hash_write(hdev, CR_INTCLEAR, reg);
 736
 737         if (reg & CR_INT_NEW_RESULTS_SET) {
 738                 dev_dbg(hdev->dev, "IRQ CR_INT_NEW_RESULTS_SET\n");
 739                 if (DRIVER_FLAGS_BUSY & hdev->flags) {
 740                         hdev->flags |= DRIVER_FLAGS_OUTPUT_READY;
 741                         if (!(DRIVER_FLAGS_CPU & hdev->flags))
 742                                 hdev->flags |= DRIVER_FLAGS_DMA_READY;
 743                         tasklet_schedule(&hdev->done_task);
 744                 } else {
 745                         dev_warn(hdev->dev,
 746                                  "HASH interrupt when no active requests.\n");
 747                 }
 748         } else if (reg & CR_INT_RESULTS_AVAILABLE) {
 749                 dev_warn(hdev->dev,
 750                          "IRQ triggered before the hash had completed\n");
 751         } else if (reg & CR_INT_RESULT_READ_ERR) {
 752                 dev_warn(hdev->dev,
 753                          "Attempt to read from an empty result queue\n");
 754         } else if (reg & CR_INT_MESSAGE_WRITE_ERROR) {
 755                 dev_warn(hdev->dev,
 756                          "Data written before the hardware was configured\n");
 757         }
 758         return IRQ_HANDLED;
 759 }
 760
 761 static struct ahash_alg img_algs[] = {
 762         {
 763                 .init = img_hash_init,
 764                 .update = img_hash_update,
 765                 .final = img_hash_final,
 766                 .finup = img_hash_finup,
 767                 .export = img_hash_export,
 768                 .import = img_hash_import,
 769                 .digest = img_hash_digest,
 770                 .halg = {
 771                         .digestsize = MD5_DIGEST_SIZE,
 772                         .statesize = sizeof(struct md5_state),
 773                         .base = {
 774                                 .cra_name = "md5",
 775                                 .cra_driver_name = "img-md5",
 776                                 .cra_priority = 300,
 777                                 .cra_flags =
 778                                 CRYPTO_ALG_ASYNC |
 779                                 CRYPTO_ALG_NEED_FALLBACK,
 780                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
 781                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 782                                 .cra_init = img_hash_cra_md5_init,
 783                                 .cra_exit = img_hash_cra_exit,
 784                                 .cra_module = THIS_MODULE,
 785                         }
 786                 }
 787         },
 788         {
 789                 .init = img_hash_init,
 790                 .update = img_hash_update,
 791                 .final = img_hash_final,
 792                 .finup = img_hash_finup,
 793                 .export = img_hash_export,
 794                 .import = img_hash_import,
 795                 .digest = img_hash_digest,
 796                 .halg = {
 797                         .digestsize = SHA1_DIGEST_SIZE,
 798                         .statesize = sizeof(struct sha1_state),
 799                         .base = {
 800                                 .cra_name = "sha1",
 801                                 .cra_driver_name = "img-sha1",
 802                                 .cra_priority = 300,
 803                                 .cra_flags =
 804                                 CRYPTO_ALG_ASYNC |
 805                                 CRYPTO_ALG_NEED_FALLBACK,
 806                                 .cra_blocksize = SHA1_BLOCK_SIZE,
 807                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 808                                 .cra_init = img_hash_cra_sha1_init,
 809                                 .cra_exit = img_hash_cra_exit,
 810                                 .cra_module = THIS_MODULE,
 811                         }
 812                 }
 813         },
 814         {
 815                 .init = img_hash_init,
 816                 .update = img_hash_update,
 817                 .final = img_hash_final,
 818                 .finup = img_hash_finup,
 819                 .export = img_hash_export,
 820                 .import = img_hash_import,
 821                 .digest = img_hash_digest,
 822                 .halg = {
 823                         .digestsize = SHA224_DIGEST_SIZE,
 824                         .statesize = sizeof(struct sha256_state),
 825                         .base = {
 826                                 .cra_name = "sha224",
 827                                 .cra_driver_name = "img-sha224",
 828                                 .cra_priority = 300,
 829                                 .cra_flags =
 830                                 CRYPTO_ALG_ASYNC |
 831                                 CRYPTO_ALG_NEED_FALLBACK,
 832                                 .cra_blocksize = SHA224_BLOCK_SIZE,
 833                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 834                                 .cra_init = img_hash_cra_sha224_init,
 835                                 .cra_exit = img_hash_cra_exit,
 836                                 .cra_module = THIS_MODULE,
 837                         }
 838                 }
 839         },
 840         {
 841                 .init = img_hash_init,
 842                 .update = img_hash_update,
 843                 .final = img_hash_final,
 844                 .finup = img_hash_finup,
 845                 .export = img_hash_export,
 846                 .import = img_hash_import,
 847                 .digest = img_hash_digest,
 848                 .halg = {
 849                         .digestsize = SHA256_DIGEST_SIZE,
 850                         .statesize = sizeof(struct sha256_state),
 851                         .base = {
 852                                 .cra_name = "sha256",
 853                                 .cra_driver_name = "img-sha256",
 854                                 .cra_priority = 300,
 855                                 .cra_flags =
 856                                 CRYPTO_ALG_ASYNC |
 857                                 CRYPTO_ALG_NEED_FALLBACK,
 858                                 .cra_blocksize = SHA256_BLOCK_SIZE,
 859                                 .cra_ctxsize = sizeof(struct img_hash_ctx),
 860                                 .cra_init = img_hash_cra_sha256_init,
 861                                 .cra_exit = img_hash_cra_exit,
 862                                 .cra_module = THIS_MODULE,
 863                         }
 864                 }
 865         }
 866 };
 867
 868 static int img_register_algs(struct img_hash_dev *hdev)
 869 {
 870         int i, err;
 871
 872         for (i = 0; i < ARRAY_SIZE(img_algs); i++) {
 873                 err = crypto_register_ahash(&img_algs[i]);
 874                 if (err)
 875                         goto err_reg;
 876         }
 877         return 0;
 878
 879 err_reg:
 880         for (; i--; )
 881                 crypto_unregister_ahash(&img_algs[i]);
 882
 883         return err;
 884 }
 885
 886 static int img_unregister_algs(struct img_hash_dev *hdev)
 887 {
 888         int i;
 889
 890         for (i = 0; i < ARRAY_SIZE(img_algs); i++)
 891                 crypto_unregister_ahash(&img_algs[i]);
 892         return 0;
 893 }
 894
 895 static void img_hash_done_task(unsigned long data)
 896 {
 897         struct img_hash_dev *hdev = (struct img_hash_dev *)data;
 898         int err = 0;
 899
 900         if (hdev->err == -EINVAL) {
 901                 err = hdev->err;
 902                 goto finish;
 903         }
 904
 905         if (!(DRIVER_FLAGS_BUSY & hdev->flags)) {
 906                 img_hash_handle_queue(hdev, NULL);
 907                 return;
 908         }
 909
 910         if (DRIVER_FLAGS_CPU & hdev->flags) {
 911                 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
 912                         hdev->flags &= ~DRIVER_FLAGS_OUTPUT_READY;
 913                         goto finish;
 914                 }
 915         } else if (DRIVER_FLAGS_DMA_READY & hdev->flags) {
 916                 if (DRIVER_FLAGS_DMA_ACTIVE & hdev->flags) {
 917                         hdev->flags &= ~DRIVER_FLAGS_DMA_ACTIVE;
 918                         img_hash_write_via_dma_stop(hdev);
 919                         if (hdev->err) {
 920                                 err = hdev->err;
 921                                 goto finish;
 922                         }
 923                 }
 924                 if (DRIVER_FLAGS_OUTPUT_READY & hdev->flags) {
 925                         hdev->flags &= ~(DRIVER_FLAGS_DMA_READY |
 926                                         DRIVER_FLAGS_OUTPUT_READY);
 927                         goto finish;
 928                 }
 929         }
 930         return;
 931
 932 finish:
 933         img_hash_finish_req(hdev->req, err);
 934 }
 935
 936 static const struct of_device_id img_hash_match[] = {
 937         { .compatible = "img,hash-accelerator" },
 938         {}
 939 };
 940 MODULE_DEVICE_TABLE(of, img_hash_match);
 941
 942 static int img_hash_probe(struct platform_device *pdev)
 943 {
 944         struct img_hash_dev *hdev;
 945         struct device *dev = &pdev->dev;
 946         struct resource *hash_res;
 947         int     irq;
 948         int err;
 949
 950         hdev = devm_kzalloc(dev, sizeof(*hdev), GFP_KERNEL);
 951         if (hdev == NULL)
 952                 return -ENOMEM;
 953
 954         spin_lock_init(&hdev->lock);
 955
 956         hdev->dev = dev;
 957
 958         platform_set_drvdata(pdev, hdev);
 959
 960         INIT_LIST_HEAD(&hdev->list);
 961
 962         tasklet_init(&hdev->done_task, img_hash_done_task, (unsigned long)hdev);
 963         tasklet_init(&hdev->dma_task, img_hash_dma_task, (unsigned long)hdev);
 964
 965         crypto_init_queue(&hdev->queue, IMG_HASH_QUEUE_LENGTH);
 966
 967         /* Register bank */
 968         hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 969
 970         hdev->io_base = devm_ioremap_resource(dev, hash_res);
 971         if (IS_ERR(hdev->io_base)) {
 972                 err = PTR_ERR(hdev->io_base);
 973                 dev_err(dev, "can't ioremap, returned %d\n", err);
 974
 975                 goto res_err;
 976         }
 977
 978         /* Write port (DMA or CPU) */
 979         hash_res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 980         hdev->cpu_addr = devm_ioremap_resource(dev, hash_res);
 981         if (IS_ERR(hdev->cpu_addr)) {
 982                 dev_err(dev, "can't ioremap write port\n");
 983                 err = PTR_ERR(hdev->cpu_addr);
 984                 goto res_err;
 985         }
 986         hdev->bus_addr = hash_res->start;
 987
 988         irq = platform_get_irq(pdev, 0);
 989         if (irq < 0) {
 990                 dev_err(dev, "no IRQ resource info\n");
 991                 err = irq;
 992                 goto res_err;
 993         }
 994
 995         err = devm_request_irq(dev, irq, img_irq_handler, 0,
 996                                dev_name(dev), hdev);
 997         if (err) {
 998                 dev_err(dev, "unable to request irq\n");
 999                 goto res_err;
1000         }
1001         dev_dbg(dev, "using IRQ channel %d\n", irq);
1002
1003         hdev->hash_clk = devm_clk_get(&pdev->dev, "hash");
1004         if (IS_ERR(hdev->hash_clk)) {
1005                 dev_err(dev, "clock initialization failed.\n");
1006                 err = PTR_ERR(hdev->hash_clk);
1007                 goto res_err;
1008         }
1009
1010         hdev->sys_clk = devm_clk_get(&pdev->dev, "sys");
1011         if (IS_ERR(hdev->sys_clk)) {
1012                 dev_err(dev, "clock initialization failed.\n");
1013                 err = PTR_ERR(hdev->sys_clk);
1014                 goto res_err;
1015         }
1016
1017         err = clk_prepare_enable(hdev->hash_clk);
1018         if (err)
1019                 goto res_err;
1020
1021         err = clk_prepare_enable(hdev->sys_clk);
1022         if (err)
1023                 goto clk_err;
1024
1025         err = img_hash_dma_init(hdev);
1026         if (err)
1027                 goto dma_err;
1028
1029         dev_dbg(dev, "using %s for DMA transfers\n",
1030                 dma_chan_name(hdev->dma_lch));
1031
1032         spin_lock(&img_hash.lock);
1033         list_add_tail(&hdev->list, &img_hash.dev_list);
1034         spin_unlock(&img_hash.lock);
1035
1036         err = img_register_algs(hdev);
1037         if (err)
1038                 goto err_algs;
1039         dev_info(dev, "Img MD5/SHA1/SHA224/SHA256 Hardware accelerator initialized\n");
1040
1041         return 0;
1042
1043 err_algs:
1044         spin_lock(&img_hash.lock);
1045         list_del(&hdev->list);
1046         spin_unlock(&img_hash.lock);
1047         dma_release_channel(hdev->dma_lch);
1048 dma_err:
1049         clk_disable_unprepare(hdev->sys_clk);
1050 clk_err:
1051         clk_disable_unprepare(hdev->hash_clk);
1052 res_err:
1053         tasklet_kill(&hdev->done_task);
1054         tasklet_kill(&hdev->dma_task);
1055
1056         return err;
1057 }
1058
1059 static int img_hash_remove(struct platform_device *pdev)
1060 {
1061         struct img_hash_dev *hdev;
1062
1063         hdev = platform_get_drvdata(pdev);
1064         spin_lock(&img_hash.lock);
1065         list_del(&hdev->list);
1066         spin_unlock(&img_hash.lock);
1067
1068         img_unregister_algs(hdev);
1069
1070         tasklet_kill(&hdev->done_task);
1071         tasklet_kill(&hdev->dma_task);
1072
1073         dma_release_channel(hdev->dma_lch);
1074
1075         clk_disable_unprepare(hdev->hash_clk);
1076         clk_disable_unprepare(hdev->sys_clk);
1077
1078         return 0;
1079 }
1080
1081 #ifdef CONFIG_PM_SLEEP
1082 static int img_hash_suspend(struct device *dev)
1083 {
1084         struct img_hash_dev *hdev = dev_get_drvdata(dev);
1085
1086         clk_disable_unprepare(hdev->hash_clk);
1087         clk_disable_unprepare(hdev->sys_clk);
1088
1089         return 0;
1090 }
1091
1092 static int img_hash_resume(struct device *dev)
1093 {
1094         struct img_hash_dev *hdev = dev_get_drvdata(dev);
1095         int ret;
1096
1097         ret = clk_prepare_enable(hdev->hash_clk);
1098         if (ret)
1099                 return ret;
1100
1101         ret = clk_prepare_enable(hdev->sys_clk);
1102         if (ret) {
1103                 clk_disable_unprepare(hdev->hash_clk);
1104                 return ret;
1105         }
1106
1107         return 0;
1108 }
1109 #endif /* CONFIG_PM_SLEEP */
1110
1111 static const struct dev_pm_ops img_hash_pm_ops = {
1112         SET_SYSTEM_SLEEP_PM_OPS(img_hash_suspend, img_hash_resume)
1113 };
1114
1115 static struct platform_driver img_hash_driver = {
1116         .probe          = img_hash_probe,
1117         .remove         = img_hash_remove,
1118         .driver         = {
1119                 .name   = "img-hash-accelerator",
1120                 .pm     = &img_hash_pm_ops,
1121                 .of_match_table = of_match_ptr(img_hash_match),
1122         }
1123 };
1124 module_platform_driver(img_hash_driver);
1125
1126 MODULE_LICENSE("GPL v2");
1127 MODULE_DESCRIPTION("Imgtec SHA1/224/256 & MD5 hw accelerator driver");
1128 MODULE_AUTHOR("Will Thomas.");
1129 MODULE_AUTHOR("James Hartley <james.hartley@imgtec.com>");