drivers/crypto/nx/nx-842-pseries.c

   1 /*
   2  * Driver for IBM Power 842 compression accelerator
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  17  *
  18  * Copyright (C) IBM Corporation, 2012
  19  *
  20  * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
  21  *          Seth Jennings <sjenning@linux.vnet.ibm.com>
  22  */
  23
  24 #include <asm/vio.h>
  25
  26 #include "nx-842.h"
  27 #include "nx_csbcpb.h" /* struct nx_csbcpb */
  28
  29 MODULE_LICENSE("GPL");
  30 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
  31 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
  32 MODULE_ALIAS_CRYPTO("842");
  33 MODULE_ALIAS_CRYPTO("842-nx");
  34
  35 static struct nx842_constraints nx842_pseries_constraints = {
  36         .alignment =    DDE_BUFFER_ALIGN,
  37         .multiple =     DDE_BUFFER_LAST_MULT,
  38         .minimum =      DDE_BUFFER_LAST_MULT,
  39         .maximum =      PAGE_SIZE, /* dynamic, max_sync_size */
  40 };
  41
  42 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
  43 {
  44         if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
  45                 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
  46                          in ? "input" : "output", buf,
  47                          nx842_pseries_constraints.alignment);
  48                 return -EINVAL;
  49         }
  50         if (*len % nx842_pseries_constraints.multiple) {
  51                 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
  52                          in ? "input" : "output", *len,
  53                          nx842_pseries_constraints.multiple);
  54                 if (in)
  55                         return -EINVAL;
  56                 *len = round_down(*len, nx842_pseries_constraints.multiple);
  57         }
  58         if (*len < nx842_pseries_constraints.minimum) {
  59                 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
  60                          in ? "input" : "output", *len,
  61                          nx842_pseries_constraints.minimum);
  62                 return -EINVAL;
  63         }
  64         if (*len > nx842_pseries_constraints.maximum) {
  65                 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
  66                          in ? "input" : "output", *len,
  67                          nx842_pseries_constraints.maximum);
  68                 if (in)
  69                         return -EINVAL;
  70                 *len = nx842_pseries_constraints.maximum;
  71         }
  72         return 0;
  73 }
  74
  75 /* I assume we need to align the CSB? */
  76 #define WORKMEM_ALIGN   (256)
  77
  78 struct nx842_workmem {
  79         /* scatterlist */
  80         char slin[4096];
  81         char slout[4096];
  82         /* coprocessor status/parameter block */
  83         struct nx_csbcpb csbcpb;
  84
  85         char padding[WORKMEM_ALIGN];
  86 } __aligned(WORKMEM_ALIGN);
  87
  88 /* Macros for fields within nx_csbcpb */
  89 /* Check the valid bit within the csbcpb valid field */
  90 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
  91
  92 /* CE macros operate on the completion_extension field bits in the csbcpb.
  93  * CE0 0=full completion, 1=partial completion
  94  * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
  95  * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
  96 #define NX842_CSBCPB_CE0(x)     (x & BIT_MASK(7))
  97 #define NX842_CSBCPB_CE1(x)     (x & BIT_MASK(6))
  98 #define NX842_CSBCPB_CE2(x)     (x & BIT_MASK(5))
  99
 100 /* The NX unit accepts data only on 4K page boundaries */
 101 #define NX842_HW_PAGE_SIZE      (4096)
 102 #define NX842_HW_PAGE_MASK      (~(NX842_HW_PAGE_SIZE-1))
 103
 104 struct ibm_nx842_counters {
 105         atomic64_t comp_complete;
 106         atomic64_t comp_failed;
 107         atomic64_t decomp_complete;
 108         atomic64_t decomp_failed;
 109         atomic64_t swdecomp;
 110         atomic64_t comp_times[32];
 111         atomic64_t decomp_times[32];
 112 };
 113
 114 static struct nx842_devdata {
 115         struct vio_dev *vdev;
 116         struct device *dev;
 117         struct ibm_nx842_counters *counters;
 118         unsigned int max_sg_len;
 119         unsigned int max_sync_size;
 120         unsigned int max_sync_sg;
 121 } __rcu *devdata;
 122 static DEFINE_SPINLOCK(devdata_mutex);
 123
 124 #define NX842_COUNTER_INC(_x) \
 125 static inline void nx842_inc_##_x( \
 126         const struct nx842_devdata *dev) { \
 127         if (dev) \
 128                 atomic64_inc(&dev->counters->_x); \
 129 }
 130 NX842_COUNTER_INC(comp_complete);
 131 NX842_COUNTER_INC(comp_failed);
 132 NX842_COUNTER_INC(decomp_complete);
 133 NX842_COUNTER_INC(decomp_failed);
 134 NX842_COUNTER_INC(swdecomp);
 135
 136 #define NX842_HIST_SLOTS 16
 137
 138 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
 139 {
 140         int bucket = fls(time);
 141
 142         if (bucket)
 143                 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
 144
 145         atomic64_inc(&times[bucket]);
 146 }
 147
 148 /* NX unit operation flags */
 149 #define NX842_OP_COMPRESS       0x0
 150 #define NX842_OP_CRC            0x1
 151 #define NX842_OP_DECOMPRESS     0x2
 152 #define NX842_OP_COMPRESS_CRC   (NX842_OP_COMPRESS | NX842_OP_CRC)
 153 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
 154 #define NX842_OP_ASYNC          (1<<23)
 155 #define NX842_OP_NOTIFY         (1<<22)
 156 #define NX842_OP_NOTIFY_INT(x)  ((x & 0xff)<<8)
 157
 158 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
 159 {
 160         /* No use of DMA mappings within the driver. */
 161         return 0;
 162 }
 163
 164 struct nx842_slentry {
 165         __be64 ptr; /* Real address (use __pa()) */
 166         __be64 len;
 167 };
 168
 169 /* pHyp scatterlist entry */
 170 struct nx842_scatterlist {
 171         int entry_nr; /* number of slentries */
 172         struct nx842_slentry *entries; /* ptr to array of slentries */
 173 };
 174
 175 /* Does not include sizeof(entry_nr) in the size */
 176 static inline unsigned long nx842_get_scatterlist_size(
 177                                 struct nx842_scatterlist *sl)
 178 {
 179         return sl->entry_nr * sizeof(struct nx842_slentry);
 180 }
 181
 182 static int nx842_build_scatterlist(unsigned long buf, int len,
 183                         struct nx842_scatterlist *sl)
 184 {
 185         unsigned long entrylen;
 186         struct nx842_slentry *entry;
 187
 188         sl->entry_nr = 0;
 189
 190         entry = sl->entries;
 191         while (len) {
 192                 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
 193                 entrylen = min_t(int, len,
 194                                  LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
 195                 entry->len = cpu_to_be64(entrylen);
 196
 197                 len -= entrylen;
 198                 buf += entrylen;
 199
 200                 sl->entry_nr++;
 201                 entry++;
 202         }
 203
 204         return 0;
 205 }
 206
 207 static int nx842_validate_result(struct device *dev,
 208         struct cop_status_block *csb)
 209 {
 210         /* The csb must be valid after returning from vio_h_cop_sync */
 211         if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
 212                 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
 213                                 __func__);
 214                 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
 215                                 csb->valid,
 216                                 csb->crb_seq_number,
 217                                 csb->completion_code,
 218                                 csb->completion_extension);
 219                 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
 220                                 be32_to_cpu(csb->processed_byte_count),
 221                                 (unsigned long)be64_to_cpu(csb->address));
 222                 return -EIO;
 223         }
 224
 225         /* Check return values from the hardware in the CSB */
 226         switch (csb->completion_code) {
 227         case 0: /* Completed without error */
 228                 break;
 229         case 64: /* Compression ok, but output larger than input */
 230                 dev_dbg(dev, "%s: output size larger than input size\n",
 231                                         __func__);
 232                 break;
 233         case 13: /* Output buffer too small */
 234                 dev_dbg(dev, "%s: Out of space in output buffer\n",
 235                                         __func__);
 236                 return -ENOSPC;
 237         case 65: /* Calculated CRC doesn't match the passed value */
 238                 dev_dbg(dev, "%s: CRC mismatch for decompression\n",
 239                                         __func__);
 240                 return -EINVAL;
 241         case 66: /* Input data contains an illegal template field */
 242         case 67: /* Template indicates data past the end of the input stream */
 243                 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
 244                                         __func__, csb->completion_code);
 245                 return -EINVAL;
 246         default:
 247                 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
 248                                         __func__, csb->completion_code);
 249                 return -EIO;
 250         }
 251
 252         /* Hardware sanity check */
 253         if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
 254                 dev_err(dev, "%s: No error returned by hardware, but "
 255                                 "data returned is unusable, contact support.\n"
 256                                 "(Additional info: csbcbp->processed bytes "
 257                                 "does not specify processed bytes for the "
 258                                 "target buffer.)\n", __func__);
 259                 return -EIO;
 260         }
 261
 262         return 0;
 263 }
 264
 265 /**
 266  * nx842_pseries_compress - Compress data using the 842 algorithm
 267  *
 268  * Compression provide by the NX842 coprocessor on IBM Power systems.
 269  * The input buffer is compressed and the result is stored in the
 270  * provided output buffer.
 271  *
 272  * Upon return from this function @outlen contains the length of the
 273  * compressed data.  If there is an error then @outlen will be 0 and an
 274  * error will be specified by the return code from this function.
 275  *
 276  * @in: Pointer to input buffer
 277  * @inlen: Length of input buffer
 278  * @out: Pointer to output buffer
 279  * @outlen: Length of output buffer
 280  * @wrkmem: ptr to buffer for working memory, size determined by
 281  *          nx842_pseries_driver.workmem_size
 282  *
 283  * Returns:
 284  *   0          Success, output of length @outlen stored in the buffer at @out
 285  *   -ENOMEM    Unable to allocate internal buffers
 286  *   -ENOSPC    Output buffer is to small
 287  *   -EIO       Internal error
 288  *   -ENODEV    Hardware unavailable
 289  */
 290 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
 291                                   unsigned char *out, unsigned int *outlen,
 292                                   void *wmem)
 293 {
 294         struct nx842_devdata *local_devdata;
 295         struct device *dev = NULL;
 296         struct nx842_workmem *workmem;
 297         struct nx842_scatterlist slin, slout;
 298         struct nx_csbcpb *csbcpb;
 299         int ret = 0, max_sync_size;
 300         unsigned long inbuf, outbuf;
 301         struct vio_pfo_op op = {
 302                 .done = NULL,
 303                 .handle = 0,
 304                 .timeout = 0,
 305         };
 306         unsigned long start = get_tb();
 307
 308         inbuf = (unsigned long)in;
 309         if (check_constraints(inbuf, &inlen, true))
 310                 return -EINVAL;
 311
 312         outbuf = (unsigned long)out;
 313         if (check_constraints(outbuf, outlen, false))
 314                 return -EINVAL;
 315
 316         rcu_read_lock();
 317         local_devdata = rcu_dereference(devdata);
 318         if (!local_devdata || !local_devdata->dev) {
 319                 rcu_read_unlock();
 320                 return -ENODEV;
 321         }
 322         max_sync_size = local_devdata->max_sync_size;
 323         dev = local_devdata->dev;
 324
 325         /* Init scatterlist */
 326         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 327         slin.entries = (struct nx842_slentry *)workmem->slin;
 328         slout.entries = (struct nx842_slentry *)workmem->slout;
 329
 330         /* Init operation */
 331         op.flags = NX842_OP_COMPRESS_CRC;
 332         csbcpb = &workmem->csbcpb;
 333         memset(csbcpb, 0, sizeof(*csbcpb));
 334         op.csbcpb = nx842_get_pa(csbcpb);
 335
 336         if ((inbuf & NX842_HW_PAGE_MASK) ==
 337             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 338                 /* Create direct DDE */
 339                 op.in = nx842_get_pa((void *)inbuf);
 340                 op.inlen = inlen;
 341         } else {
 342                 /* Create indirect DDE (scatterlist) */
 343                 nx842_build_scatterlist(inbuf, inlen, &slin);
 344                 op.in = nx842_get_pa(slin.entries);
 345                 op.inlen = -nx842_get_scatterlist_size(&slin);
 346         }
 347
 348         if ((outbuf & NX842_HW_PAGE_MASK) ==
 349             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 350                 /* Create direct DDE */
 351                 op.out = nx842_get_pa((void *)outbuf);
 352                 op.outlen = *outlen;
 353         } else {
 354                 /* Create indirect DDE (scatterlist) */
 355                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 356                 op.out = nx842_get_pa(slout.entries);
 357                 op.outlen = -nx842_get_scatterlist_size(&slout);
 358         }
 359
 360         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 361                 __func__, (unsigned long)op.in, (long)op.inlen,
 362                 (unsigned long)op.out, (long)op.outlen);
 363
 364         /* Send request to pHyp */
 365         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 366
 367         /* Check for pHyp error */
 368         if (ret) {
 369                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 370                         __func__, ret, op.hcall_err);
 371                 ret = -EIO;
 372                 goto unlock;
 373         }
 374
 375         /* Check for hardware error */
 376         ret = nx842_validate_result(dev, &csbcpb->csb);
 377         if (ret)
 378                 goto unlock;
 379
 380         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 381         dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
 382
 383 unlock:
 384         if (ret)
 385                 nx842_inc_comp_failed(local_devdata);
 386         else {
 387                 nx842_inc_comp_complete(local_devdata);
 388                 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
 389                         (get_tb() - start) / tb_ticks_per_usec);
 390         }
 391         rcu_read_unlock();
 392         return ret;
 393 }
 394
 395 /**
 396  * nx842_pseries_decompress - Decompress data using the 842 algorithm
 397  *
 398  * Decompression provide by the NX842 coprocessor on IBM Power systems.
 399  * The input buffer is decompressed and the result is stored in the
 400  * provided output buffer.  The size allocated to the output buffer is
 401  * provided by the caller of this function in @outlen.  Upon return from
 402  * this function @outlen contains the length of the decompressed data.
 403  * If there is an error then @outlen will be 0 and an error will be
 404  * specified by the return code from this function.
 405  *
 406  * @in: Pointer to input buffer
 407  * @inlen: Length of input buffer
 408  * @out: Pointer to output buffer
 409  * @outlen: Length of output buffer
 410  * @wrkmem: ptr to buffer for working memory, size determined by
 411  *          nx842_pseries_driver.workmem_size
 412  *
 413  * Returns:
 414  *   0          Success, output of length @outlen stored in the buffer at @out
 415  *   -ENODEV    Hardware decompression device is unavailable
 416  *   -ENOMEM    Unable to allocate internal buffers
 417  *   -ENOSPC    Output buffer is to small
 418  *   -EINVAL    Bad input data encountered when attempting decompress
 419  *   -EIO       Internal error
 420  */
 421 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
 422                                     unsigned char *out, unsigned int *outlen,
 423                                     void *wmem)
 424 {
 425         struct nx842_devdata *local_devdata;
 426         struct device *dev = NULL;
 427         struct nx842_workmem *workmem;
 428         struct nx842_scatterlist slin, slout;
 429         struct nx_csbcpb *csbcpb;
 430         int ret = 0, max_sync_size;
 431         unsigned long inbuf, outbuf;
 432         struct vio_pfo_op op = {
 433                 .done = NULL,
 434                 .handle = 0,
 435                 .timeout = 0,
 436         };
 437         unsigned long start = get_tb();
 438
 439         /* Ensure page alignment and size */
 440         inbuf = (unsigned long)in;
 441         if (check_constraints(inbuf, &inlen, true))
 442                 return -EINVAL;
 443
 444         outbuf = (unsigned long)out;
 445         if (check_constraints(outbuf, outlen, false))
 446                 return -EINVAL;
 447
 448         rcu_read_lock();
 449         local_devdata = rcu_dereference(devdata);
 450         if (!local_devdata || !local_devdata->dev) {
 451                 rcu_read_unlock();
 452                 return -ENODEV;
 453         }
 454         max_sync_size = local_devdata->max_sync_size;
 455         dev = local_devdata->dev;
 456
 457         workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
 458
 459         /* Init scatterlist */
 460         slin.entries = (struct nx842_slentry *)workmem->slin;
 461         slout.entries = (struct nx842_slentry *)workmem->slout;
 462
 463         /* Init operation */
 464         op.flags = NX842_OP_DECOMPRESS_CRC;
 465         csbcpb = &workmem->csbcpb;
 466         memset(csbcpb, 0, sizeof(*csbcpb));
 467         op.csbcpb = nx842_get_pa(csbcpb);
 468
 469         if ((inbuf & NX842_HW_PAGE_MASK) ==
 470             ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
 471                 /* Create direct DDE */
 472                 op.in = nx842_get_pa((void *)inbuf);
 473                 op.inlen = inlen;
 474         } else {
 475                 /* Create indirect DDE (scatterlist) */
 476                 nx842_build_scatterlist(inbuf, inlen, &slin);
 477                 op.in = nx842_get_pa(slin.entries);
 478                 op.inlen = -nx842_get_scatterlist_size(&slin);
 479         }
 480
 481         if ((outbuf & NX842_HW_PAGE_MASK) ==
 482             ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
 483                 /* Create direct DDE */
 484                 op.out = nx842_get_pa((void *)outbuf);
 485                 op.outlen = *outlen;
 486         } else {
 487                 /* Create indirect DDE (scatterlist) */
 488                 nx842_build_scatterlist(outbuf, *outlen, &slout);
 489                 op.out = nx842_get_pa(slout.entries);
 490                 op.outlen = -nx842_get_scatterlist_size(&slout);
 491         }
 492
 493         dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
 494                 __func__, (unsigned long)op.in, (long)op.inlen,
 495                 (unsigned long)op.out, (long)op.outlen);
 496
 497         /* Send request to pHyp */
 498         ret = vio_h_cop_sync(local_devdata->vdev, &op);
 499
 500         /* Check for pHyp error */
 501         if (ret) {
 502                 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
 503                         __func__, ret, op.hcall_err);
 504                 goto unlock;
 505         }
 506
 507         /* Check for hardware error */
 508         ret = nx842_validate_result(dev, &csbcpb->csb);
 509         if (ret)
 510                 goto unlock;
 511
 512         *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
 513
 514 unlock:
 515         if (ret)
 516                 /* decompress fail */
 517                 nx842_inc_decomp_failed(local_devdata);
 518         else {
 519                 nx842_inc_decomp_complete(local_devdata);
 520                 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
 521                         (get_tb() - start) / tb_ticks_per_usec);
 522         }
 523
 524         rcu_read_unlock();
 525         return ret;
 526 }
 527
 528 /**
 529  * nx842_OF_set_defaults -- Set default (disabled) values for devdata
 530  *
 531  * @devdata - struct nx842_devdata to update
 532  *
 533  * Returns:
 534  *  0 on success
 535  *  -ENOENT if @devdata ptr is NULL
 536  */
 537 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
 538 {
 539         if (devdata) {
 540                 devdata->max_sync_size = 0;
 541                 devdata->max_sync_sg = 0;
 542                 devdata->max_sg_len = 0;
 543                 return 0;
 544         } else
 545                 return -ENOENT;
 546 }
 547
 548 /**
 549  * nx842_OF_upd_status -- Check the device info from OF status prop
 550  *
 551  * The status property indicates if the accelerator is enabled.  If the
 552  * device is in the OF tree it indicates that the hardware is present.
 553  * The status field indicates if the device is enabled when the status
 554  * is 'okay'.  Otherwise the device driver will be disabled.
 555  *
 556  * @devdata: struct nx842_devdata to use for dev_info
 557  * @prop: struct property point containing the maxsyncop for the update
 558  *
 559  * Returns:
 560  *  0 - Device is available
 561  *  -ENODEV - Device is not available
 562  */
 563 static int nx842_OF_upd_status(struct nx842_devdata *devdata,
 564                                struct property *prop)
 565 {
 566         const char *status = (const char *)prop->value;
 567
 568         if (!strncmp(status, "okay", (size_t)prop->length))
 569                 return 0;
 570         if (!strncmp(status, "disabled", (size_t)prop->length))
 571                 return -ENODEV;
 572         dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
 573
 574         return -EINVAL;
 575 }
 576
 577 /**
 578  * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
 579  *
 580  * Definition of the 'ibm,max-sg-len' OF property:
 581  *  This field indicates the maximum byte length of a scatter list
 582  *  for the platform facility. It is a single cell encoded as with encode-int.
 583  *
 584  * Example:
 585  *  # od -x ibm,max-sg-len
 586  *  0000000 0000 0ff0
 587  *
 588  *  In this example, the maximum byte length of a scatter list is
 589  *  0x0ff0 (4,080).
 590  *
 591  * @devdata - struct nx842_devdata to update
 592  * @prop - struct property point containing the maxsyncop for the update
 593  *
 594  * Returns:
 595  *  0 on success
 596  *  -EINVAL on failure
 597  */
 598 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
 599                                         struct property *prop) {
 600         int ret = 0;
 601         const unsigned int maxsglen = of_read_number(prop->value, 1);
 602
 603         if (prop->length != sizeof(maxsglen)) {
 604                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
 605                 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
 606                                 prop->length, sizeof(maxsglen));
 607                 ret = -EINVAL;
 608         } else {
 609                 devdata->max_sg_len = min_t(unsigned int,
 610                                             maxsglen, NX842_HW_PAGE_SIZE);
 611         }
 612
 613         return ret;
 614 }
 615
 616 /**
 617  * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
 618  *
 619  * Definition of the 'ibm,max-sync-cop' OF property:
 620  *  Two series of cells.  The first series of cells represents the maximums
 621  *  that can be synchronously compressed. The second series of cells
 622  *  represents the maximums that can be synchronously decompressed.
 623  *  1. The first cell in each series contains the count of the number of
 624  *     data length, scatter list elements pairs that follow – each being
 625  *     of the form
 626  *    a. One cell data byte length
 627  *    b. One cell total number of scatter list elements
 628  *
 629  * Example:
 630  *  # od -x ibm,max-sync-cop
 631  *  0000000 0000 0001 0000 1000 0000 01fe 0000 0001
 632  *  0000020 0000 1000 0000 01fe
 633  *
 634  *  In this example, compression supports 0x1000 (4,096) data byte length
 635  *  and 0x1fe (510) total scatter list elements.  Decompression supports
 636  *  0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
 637  *  elements.
 638  *
 639  * @devdata - struct nx842_devdata to update
 640  * @prop - struct property point containing the maxsyncop for the update
 641  *
 642  * Returns:
 643  *  0 on success
 644  *  -EINVAL on failure
 645  */
 646 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
 647                                         struct property *prop) {
 648         int ret = 0;
 649         unsigned int comp_data_limit, decomp_data_limit;
 650         unsigned int comp_sg_limit, decomp_sg_limit;
 651         const struct maxsynccop_t {
 652                 __be32 comp_elements;
 653                 __be32 comp_data_limit;
 654                 __be32 comp_sg_limit;
 655                 __be32 decomp_elements;
 656                 __be32 decomp_data_limit;
 657                 __be32 decomp_sg_limit;
 658         } *maxsynccop;
 659
 660         if (prop->length != sizeof(*maxsynccop)) {
 661                 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
 662                 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
 663                                 sizeof(*maxsynccop));
 664                 ret = -EINVAL;
 665                 goto out;
 666         }
 667
 668         maxsynccop = (const struct maxsynccop_t *)prop->value;
 669         comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
 670         comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
 671         decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
 672         decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
 673
 674         /* Use one limit rather than separate limits for compression and
 675          * decompression. Set a maximum for this so as not to exceed the
 676          * size that the header can support and round the value down to
 677          * the hardware page size (4K) */
 678         devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
 679
 680         devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
 681                                         65536);
 682
 683         if (devdata->max_sync_size < 4096) {
 684                 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
 685                                 "less than the driver minimum, unable to use "
 686                                 "the hardware device\n",
 687                                 __func__, devdata->max_sync_size);
 688                 ret = -EINVAL;
 689                 goto out;
 690         }
 691
 692         nx842_pseries_constraints.maximum = devdata->max_sync_size;
 693
 694         devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
 695         if (devdata->max_sync_sg < 1) {
 696                 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
 697                                 "less than the driver minimum, unable to use "
 698                                 "the hardware device\n",
 699                                 __func__, devdata->max_sync_sg);
 700                 ret = -EINVAL;
 701                 goto out;
 702         }
 703
 704 out:
 705         return ret;
 706 }
 707
 708 /**
 709  *
 710  * nx842_OF_upd -- Handle OF properties updates for the device.
 711  *
 712  * Set all properties from the OF tree.  Optionally, a new property
 713  * can be provided by the @new_prop pointer to overwrite an existing value.
 714  * The device will remain disabled until all values are valid, this function
 715  * will return an error for updates unless all values are valid.
 716  *
 717  * @new_prop: If not NULL, this property is being updated.  If NULL, update
 718  *  all properties from the current values in the OF tree.
 719  *
 720  * Returns:
 721  *  0 - Success
 722  *  -ENOMEM - Could not allocate memory for new devdata structure
 723  *  -EINVAL - property value not found, new_prop is not a recognized
 724  *      property for the device or property value is not valid.
 725  *  -ENODEV - Device is not available
 726  */
 727 static int nx842_OF_upd(struct property *new_prop)
 728 {
 729         struct nx842_devdata *old_devdata = NULL;
 730         struct nx842_devdata *new_devdata = NULL;
 731         struct device_node *of_node = NULL;
 732         struct property *status = NULL;
 733         struct property *maxsglen = NULL;
 734         struct property *maxsyncop = NULL;
 735         int ret = 0;
 736         unsigned long flags;
 737
 738         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 739         if (!new_devdata)
 740                 return -ENOMEM;
 741
 742         spin_lock_irqsave(&devdata_mutex, flags);
 743         old_devdata = rcu_dereference_check(devdata,
 744                         lockdep_is_held(&devdata_mutex));
 745         if (old_devdata)
 746                 of_node = old_devdata->dev->of_node;
 747
 748         if (!old_devdata || !of_node) {
 749                 pr_err("%s: device is not available\n", __func__);
 750                 spin_unlock_irqrestore(&devdata_mutex, flags);
 751                 kfree(new_devdata);
 752                 return -ENODEV;
 753         }
 754
 755         memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
 756         new_devdata->counters = old_devdata->counters;
 757
 758         /* Set ptrs for existing properties */
 759         status = of_find_property(of_node, "status", NULL);
 760         maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
 761         maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
 762         if (!status || !maxsglen || !maxsyncop) {
 763                 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
 764                 ret = -EINVAL;
 765                 goto error_out;
 766         }
 767
 768         /*
 769          * If this is a property update, there are only certain properties that
 770          * we care about. Bail if it isn't in the below list
 771          */
 772         if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
 773                          strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
 774                          strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
 775                 goto out;
 776
 777         /* Perform property updates */
 778         ret = nx842_OF_upd_status(new_devdata, status);
 779         if (ret)
 780                 goto error_out;
 781
 782         ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
 783         if (ret)
 784                 goto error_out;
 785
 786         ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
 787         if (ret)
 788                 goto error_out;
 789
 790 out:
 791         dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
 792                         __func__, new_devdata->max_sync_size,
 793                         old_devdata->max_sync_size);
 794         dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
 795                         __func__, new_devdata->max_sync_sg,
 796                         old_devdata->max_sync_sg);
 797         dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
 798                         __func__, new_devdata->max_sg_len,
 799                         old_devdata->max_sg_len);
 800
 801         rcu_assign_pointer(devdata, new_devdata);
 802         spin_unlock_irqrestore(&devdata_mutex, flags);
 803         synchronize_rcu();
 804         dev_set_drvdata(new_devdata->dev, new_devdata);
 805         kfree(old_devdata);
 806         return 0;
 807
 808 error_out:
 809         if (new_devdata) {
 810                 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
 811                 nx842_OF_set_defaults(new_devdata);
 812                 rcu_assign_pointer(devdata, new_devdata);
 813                 spin_unlock_irqrestore(&devdata_mutex, flags);
 814                 synchronize_rcu();
 815                 dev_set_drvdata(new_devdata->dev, new_devdata);
 816                 kfree(old_devdata);
 817         } else {
 818                 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
 819                 spin_unlock_irqrestore(&devdata_mutex, flags);
 820         }
 821
 822         if (!ret)
 823                 ret = -EINVAL;
 824         return ret;
 825 }
 826
 827 /**
 828  * nx842_OF_notifier - Process updates to OF properties for the device
 829  *
 830  * @np: notifier block
 831  * @action: notifier action
 832  * @update: struct pSeries_reconfig_prop_update pointer if action is
 833  *      PSERIES_UPDATE_PROPERTY
 834  *
 835  * Returns:
 836  *      NOTIFY_OK on success
 837  *      NOTIFY_BAD encoded with error number on failure, use
 838  *              notifier_to_errno() to decode this value
 839  */
 840 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
 841                              void *data)
 842 {
 843         struct of_reconfig_data *upd = data;
 844         struct nx842_devdata *local_devdata;
 845         struct device_node *node = NULL;
 846
 847         rcu_read_lock();
 848         local_devdata = rcu_dereference(devdata);
 849         if (local_devdata)
 850                 node = local_devdata->dev->of_node;
 851
 852         if (local_devdata &&
 853                         action == OF_RECONFIG_UPDATE_PROPERTY &&
 854                         !strcmp(upd->dn->name, node->name)) {
 855                 rcu_read_unlock();
 856                 nx842_OF_upd(upd->prop);
 857         } else
 858                 rcu_read_unlock();
 859
 860         return NOTIFY_OK;
 861 }
 862
 863 static struct notifier_block nx842_of_nb = {
 864         .notifier_call = nx842_OF_notifier,
 865 };
 866
 867 #define nx842_counter_read(_name)                                       \
 868 static ssize_t nx842_##_name##_show(struct device *dev,         \
 869                 struct device_attribute *attr,                          \
 870                 char *buf) {                                            \
 871         struct nx842_devdata *local_devdata;                    \
 872         int p = 0;                                                      \
 873         rcu_read_lock();                                                \
 874         local_devdata = rcu_dereference(devdata);                       \
 875         if (local_devdata)                                              \
 876                 p = snprintf(buf, PAGE_SIZE, "%ld\n",                   \
 877                        atomic64_read(&local_devdata->counters->_name)); \
 878         rcu_read_unlock();                                              \
 879         return p;                                                       \
 880 }
 881
 882 #define NX842DEV_COUNTER_ATTR_RO(_name)                                 \
 883         nx842_counter_read(_name);                                      \
 884         static struct device_attribute dev_attr_##_name = __ATTR(_name, \
 885                                                 0444,                   \
 886                                                 nx842_##_name##_show,\
 887                                                 NULL);
 888
 889 NX842DEV_COUNTER_ATTR_RO(comp_complete);
 890 NX842DEV_COUNTER_ATTR_RO(comp_failed);
 891 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
 892 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
 893 NX842DEV_COUNTER_ATTR_RO(swdecomp);
 894
 895 static ssize_t nx842_timehist_show(struct device *,
 896                 struct device_attribute *, char *);
 897
 898 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
 899                 nx842_timehist_show, NULL);
 900 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
 901                 0444, nx842_timehist_show, NULL);
 902
 903 static ssize_t nx842_timehist_show(struct device *dev,
 904                 struct device_attribute *attr, char *buf) {
 905         char *p = buf;
 906         struct nx842_devdata *local_devdata;
 907         atomic64_t *times;
 908         int bytes_remain = PAGE_SIZE;
 909         int bytes;
 910         int i;
 911
 912         rcu_read_lock();
 913         local_devdata = rcu_dereference(devdata);
 914         if (!local_devdata) {
 915                 rcu_read_unlock();
 916                 return 0;
 917         }
 918
 919         if (attr == &dev_attr_comp_times)
 920                 times = local_devdata->counters->comp_times;
 921         else if (attr == &dev_attr_decomp_times)
 922                 times = local_devdata->counters->decomp_times;
 923         else {
 924                 rcu_read_unlock();
 925                 return 0;
 926         }
 927
 928         for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
 929                 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
 930                                i ? (2<<(i-1)) : 0, (2<<i)-1,
 931                                atomic64_read(&times[i]));
 932                 bytes_remain -= bytes;
 933                 p += bytes;
 934         }
 935         /* The last bucket holds everything over
 936          * 2<<(NX842_HIST_SLOTS - 2) us */
 937         bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
 938                         2<<(NX842_HIST_SLOTS - 2),
 939                         atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
 940         p += bytes;
 941
 942         rcu_read_unlock();
 943         return p - buf;
 944 }
 945
 946 static struct attribute *nx842_sysfs_entries[] = {
 947         &dev_attr_comp_complete.attr,
 948         &dev_attr_comp_failed.attr,
 949         &dev_attr_decomp_complete.attr,
 950         &dev_attr_decomp_failed.attr,
 951         &dev_attr_swdecomp.attr,
 952         &dev_attr_comp_times.attr,
 953         &dev_attr_decomp_times.attr,
 954         NULL,
 955 };
 956
 957 static struct attribute_group nx842_attribute_group = {
 958         .name = NULL,           /* put in device directory */
 959         .attrs = nx842_sysfs_entries,
 960 };
 961
 962 static struct nx842_driver nx842_pseries_driver = {
 963         .name =         KBUILD_MODNAME,
 964         .owner =        THIS_MODULE,
 965         .workmem_size = sizeof(struct nx842_workmem),
 966         .constraints =  &nx842_pseries_constraints,
 967         .compress =     nx842_pseries_compress,
 968         .decompress =   nx842_pseries_decompress,
 969 };
 970
 971 static int nx842_pseries_crypto_init(struct crypto_tfm *tfm)
 972 {
 973         return nx842_crypto_init(tfm, &nx842_pseries_driver);
 974 }
 975
 976 static struct crypto_alg nx842_pseries_alg = {
 977         .cra_name               = "842",
 978         .cra_driver_name        = "842-nx",
 979         .cra_priority           = 300,
 980         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
 981         .cra_ctxsize            = sizeof(struct nx842_crypto_ctx),
 982         .cra_module             = THIS_MODULE,
 983         .cra_init               = nx842_pseries_crypto_init,
 984         .cra_exit               = nx842_crypto_exit,
 985         .cra_u                  = { .compress = {
 986         .coa_compress           = nx842_crypto_compress,
 987         .coa_decompress         = nx842_crypto_decompress } }
 988 };
 989
 990 static int nx842_probe(struct vio_dev *viodev,
 991                        const struct vio_device_id *id)
 992 {
 993         struct nx842_devdata *old_devdata, *new_devdata = NULL;
 994         unsigned long flags;
 995         int ret = 0;
 996
 997         new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
 998         if (!new_devdata)
 999                 return -ENOMEM;
1000
1001         new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
1002                         GFP_NOFS);
1003         if (!new_devdata->counters) {
1004                 kfree(new_devdata);
1005                 return -ENOMEM;
1006         }
1007
1008         spin_lock_irqsave(&devdata_mutex, flags);
1009         old_devdata = rcu_dereference_check(devdata,
1010                         lockdep_is_held(&devdata_mutex));
1011
1012         if (old_devdata && old_devdata->vdev != NULL) {
1013                 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
1014                 ret = -1;
1015                 goto error_unlock;
1016         }
1017
1018         dev_set_drvdata(&viodev->dev, NULL);
1019
1020         new_devdata->vdev = viodev;
1021         new_devdata->dev = &viodev->dev;
1022         nx842_OF_set_defaults(new_devdata);
1023
1024         rcu_assign_pointer(devdata, new_devdata);
1025         spin_unlock_irqrestore(&devdata_mutex, flags);
1026         synchronize_rcu();
1027         kfree(old_devdata);
1028
1029         of_reconfig_notifier_register(&nx842_of_nb);
1030
1031         ret = nx842_OF_upd(NULL);
1032         if (ret)
1033                 goto error;
1034
1035         ret = crypto_register_alg(&nx842_pseries_alg);
1036         if (ret) {
1037                 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
1038                 goto error;
1039         }
1040
1041         rcu_read_lock();
1042         dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1043         rcu_read_unlock();
1044
1045         if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1046                 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1047                 ret = -1;
1048                 goto error;
1049         }
1050
1051         return 0;
1052
1053 error_unlock:
1054         spin_unlock_irqrestore(&devdata_mutex, flags);
1055         if (new_devdata)
1056                 kfree(new_devdata->counters);
1057         kfree(new_devdata);
1058 error:
1059         return ret;
1060 }
1061
1062 static int nx842_remove(struct vio_dev *viodev)
1063 {
1064         struct nx842_devdata *old_devdata;
1065         unsigned long flags;
1066
1067         pr_info("Removing IBM Power 842 compression device\n");
1068         sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1069
1070         crypto_unregister_alg(&nx842_pseries_alg);
1071
1072         spin_lock_irqsave(&devdata_mutex, flags);
1073         old_devdata = rcu_dereference_check(devdata,
1074                         lockdep_is_held(&devdata_mutex));
1075         of_reconfig_notifier_unregister(&nx842_of_nb);
1076         RCU_INIT_POINTER(devdata, NULL);
1077         spin_unlock_irqrestore(&devdata_mutex, flags);
1078         synchronize_rcu();
1079         dev_set_drvdata(&viodev->dev, NULL);
1080         if (old_devdata)
1081                 kfree(old_devdata->counters);
1082         kfree(old_devdata);
1083
1084         return 0;
1085 }
1086
1087 static const struct vio_device_id nx842_vio_driver_ids[] = {
1088         {"ibm,compression-v1", "ibm,compression"},
1089         {"", ""},
1090 };
1091 MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);
1092
1093 static struct vio_driver nx842_vio_driver = {
1094         .name = KBUILD_MODNAME,
1095         .probe = nx842_probe,
1096         .remove = nx842_remove,
1097         .get_desired_dma = nx842_get_desired_dma,
1098         .id_table = nx842_vio_driver_ids,
1099 };
1100
1101 static int __init nx842_pseries_init(void)
1102 {
1103         struct nx842_devdata *new_devdata;
1104         int ret;
1105
1106         if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
1107                 return -ENODEV;
1108
1109         RCU_INIT_POINTER(devdata, NULL);
1110         new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1111         if (!new_devdata)
1112                 return -ENOMEM;
1113
1114         RCU_INIT_POINTER(devdata, new_devdata);
1115
1116         ret = vio_register_driver(&nx842_vio_driver);
1117         if (ret) {
1118                 pr_err("Could not register VIO driver %d\n", ret);
1119
1120                 kfree(new_devdata);
1121                 return ret;
1122         }
1123
1124         return 0;
1125 }
1126
1127 module_init(nx842_pseries_init);
1128
1129 static void __exit nx842_pseries_exit(void)
1130 {
1131         struct nx842_devdata *old_devdata;
1132         unsigned long flags;
1133
1134         crypto_unregister_alg(&nx842_pseries_alg);
1135
1136         spin_lock_irqsave(&devdata_mutex, flags);
1137         old_devdata = rcu_dereference_check(devdata,
1138                         lockdep_is_held(&devdata_mutex));
1139         RCU_INIT_POINTER(devdata, NULL);
1140         spin_unlock_irqrestore(&devdata_mutex, flags);
1141         synchronize_rcu();
1142         if (old_devdata && old_devdata->dev)
1143                 dev_set_drvdata(old_devdata->dev, NULL);
1144         kfree(old_devdata);
1145         vio_unregister_driver(&nx842_vio_driver);
1146 }
1147
1148 module_exit(nx842_pseries_exit);
1149