drivers/misc/cxl/guest.c

   1 /*
   2  * Copyright 2015 IBM Corp.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; either version
   7  * 2 of the License, or (at your option) any later version.
   8  */
   9
  10 #include <linux/spinlock.h>
  11 #include <linux/uaccess.h>
  12 #include <linux/delay.h>
  13
  14 #include "cxl.h"
  15 #include "hcalls.h"
  16 #include "trace.h"
  17
  18 #define CXL_ERROR_DETECTED_EVENT        1
  19 #define CXL_SLOT_RESET_EVENT            2
  20 #define CXL_RESUME_EVENT                3
  21
  22 static void pci_error_handlers(struct cxl_afu *afu,
  23                                 int bus_error_event,
  24                                 pci_channel_state_t state)
  25 {
  26         struct pci_dev *afu_dev;
  27
  28         if (afu->phb == NULL)
  29                 return;
  30
  31         list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {
  32                 if (!afu_dev->driver)
  33                         continue;
  34
  35                 switch (bus_error_event) {
  36                 case CXL_ERROR_DETECTED_EVENT:
  37                         afu_dev->error_state = state;
  38
  39                         if (afu_dev->driver->err_handler &&
  40                             afu_dev->driver->err_handler->error_detected)
  41                                 afu_dev->driver->err_handler->error_detected(afu_dev, state);
  42                 break;
  43                 case CXL_SLOT_RESET_EVENT:
  44                         afu_dev->error_state = state;
  45
  46                         if (afu_dev->driver->err_handler &&
  47                             afu_dev->driver->err_handler->slot_reset)
  48                                 afu_dev->driver->err_handler->slot_reset(afu_dev);
  49                 break;
  50                 case CXL_RESUME_EVENT:
  51                         if (afu_dev->driver->err_handler &&
  52                             afu_dev->driver->err_handler->resume)
  53                                 afu_dev->driver->err_handler->resume(afu_dev);
  54                 break;
  55                 }
  56         }
  57 }
  58
  59 static irqreturn_t guest_handle_psl_slice_error(struct cxl_context *ctx, u64 dsisr,
  60                                         u64 errstat)
  61 {
  62         pr_devel("in %s\n", __func__);
  63         dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%.16llx\n", errstat);
  64
  65         return cxl_ops->ack_irq(ctx, 0, errstat);
  66 }
  67
  68 static ssize_t guest_collect_vpd(struct cxl *adapter, struct cxl_afu *afu,
  69                         void *buf, size_t len)
  70 {
  71         unsigned int entries, mod;
  72         unsigned long **vpd_buf = NULL;
  73         struct sg_list *le;
  74         int rc = 0, i, tocopy;
  75         u64 out = 0;
  76
  77         if (buf == NULL)
  78                 return -EINVAL;
  79
  80         /* number of entries in the list */
  81         entries = len / SG_BUFFER_SIZE;
  82         mod = len % SG_BUFFER_SIZE;
  83         if (mod)
  84                 entries++;
  85
  86         if (entries > SG_MAX_ENTRIES) {
  87                 entries = SG_MAX_ENTRIES;
  88                 len = SG_MAX_ENTRIES * SG_BUFFER_SIZE;
  89                 mod = 0;
  90         }
  91
  92         vpd_buf = kzalloc(entries * sizeof(unsigned long *), GFP_KERNEL);
  93         if (!vpd_buf)
  94                 return -ENOMEM;
  95
  96         le = (struct sg_list *)get_zeroed_page(GFP_KERNEL);
  97         if (!le) {
  98                 rc = -ENOMEM;
  99                 goto err1;
 100         }
 101
 102         for (i = 0; i < entries; i++) {
 103                 vpd_buf[i] = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 104                 if (!vpd_buf[i]) {
 105                         rc = -ENOMEM;
 106                         goto err2;
 107                 }
 108                 le[i].phys_addr = cpu_to_be64(virt_to_phys(vpd_buf[i]));
 109                 le[i].len = cpu_to_be64(SG_BUFFER_SIZE);
 110                 if ((i == (entries - 1)) && mod)
 111                         le[i].len = cpu_to_be64(mod);
 112         }
 113
 114         if (adapter)
 115                 rc = cxl_h_collect_vpd_adapter(adapter->guest->handle,
 116                                         virt_to_phys(le), entries, &out);
 117         else
 118                 rc = cxl_h_collect_vpd(afu->guest->handle, 0,
 119                                 virt_to_phys(le), entries, &out);
 120         pr_devel("length of available (entries: %i), vpd: %#llx\n",
 121                 entries, out);
 122
 123         if (!rc) {
 124                 /*
 125                  * hcall returns in 'out' the size of available VPDs.
 126                  * It fills the buffer with as much data as possible.
 127                  */
 128                 if (out < len)
 129                         len = out;
 130                 rc = len;
 131                 if (out) {
 132                         for (i = 0; i < entries; i++) {
 133                                 if (len < SG_BUFFER_SIZE)
 134                                         tocopy = len;
 135                                 else
 136                                         tocopy = SG_BUFFER_SIZE;
 137                                 memcpy(buf, vpd_buf[i], tocopy);
 138                                 buf += tocopy;
 139                                 len -= tocopy;
 140                         }
 141                 }
 142         }
 143 err2:
 144         for (i = 0; i < entries; i++) {
 145                 if (vpd_buf[i])
 146                         free_page((unsigned long) vpd_buf[i]);
 147         }
 148         free_page((unsigned long) le);
 149 err1:
 150         kfree(vpd_buf);
 151         return rc;
 152 }
 153
 154 static int guest_get_irq_info(struct cxl_context *ctx, struct cxl_irq_info *info)
 155 {
 156         return cxl_h_collect_int_info(ctx->afu->guest->handle, ctx->process_token, info);
 157 }
 158
 159 static irqreturn_t guest_psl_irq(int irq, void *data)
 160 {
 161         struct cxl_context *ctx = data;
 162         struct cxl_irq_info irq_info;
 163         int rc;
 164
 165         pr_devel("%d: received PSL interrupt %i\n", ctx->pe, irq);
 166         rc = guest_get_irq_info(ctx, &irq_info);
 167         if (rc) {
 168                 WARN(1, "Unable to get IRQ info: %i\n", rc);
 169                 return IRQ_HANDLED;
 170         }
 171
 172         rc = cxl_irq_psl8(irq, ctx, &irq_info);
 173         return rc;
 174 }
 175
 176 static int afu_read_error_state(struct cxl_afu *afu, int *state_out)
 177 {
 178         u64 state;
 179         int rc = 0;
 180
 181         if (!afu)
 182                 return -EIO;
 183
 184         rc = cxl_h_read_error_state(afu->guest->handle, &state);
 185         if (!rc) {
 186                 WARN_ON(state != H_STATE_NORMAL &&
 187                         state != H_STATE_DISABLE &&
 188                         state != H_STATE_TEMP_UNAVAILABLE &&
 189                         state != H_STATE_PERM_UNAVAILABLE);
 190                 *state_out = state & 0xffffffff;
 191         }
 192         return rc;
 193 }
 194
 195 static irqreturn_t guest_slice_irq_err(int irq, void *data)
 196 {
 197         struct cxl_afu *afu = data;
 198         int rc;
 199         u64 serr, afu_error, dsisr;
 200
 201         rc = cxl_h_get_fn_error_interrupt(afu->guest->handle, &serr);
 202         if (rc) {
 203                 dev_crit(&afu->dev, "Couldn't read PSL_SERR_An: %d\n", rc);
 204                 return IRQ_HANDLED;
 205         }
 206         afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An);
 207         dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
 208         cxl_afu_decode_psl_serr(afu, serr);
 209         dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error);
 210         dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr);
 211
 212         rc = cxl_h_ack_fn_error_interrupt(afu->guest->handle, serr);
 213         if (rc)
 214                 dev_crit(&afu->dev, "Couldn't ack slice error interrupt: %d\n",
 215                         rc);
 216
 217         return IRQ_HANDLED;
 218 }
 219
 220
 221 static int irq_alloc_range(struct cxl *adapter, int len, int *irq)
 222 {
 223         int i, n;
 224         struct irq_avail *cur;
 225
 226         for (i = 0; i < adapter->guest->irq_nranges; i++) {
 227                 cur = &adapter->guest->irq_avail[i];
 228                 n = bitmap_find_next_zero_area(cur->bitmap, cur->range,
 229                                         0, len, 0);
 230                 if (n < cur->range) {
 231                         bitmap_set(cur->bitmap, n, len);
 232                         *irq = cur->offset + n;
 233                         pr_devel("guest: allocate IRQs %#x->%#x\n",
 234                                 *irq, *irq + len - 1);
 235
 236                         return 0;
 237                 }
 238         }
 239         return -ENOSPC;
 240 }
 241
 242 static int irq_free_range(struct cxl *adapter, int irq, int len)
 243 {
 244         int i, n;
 245         struct irq_avail *cur;
 246
 247         if (len == 0)
 248                 return -ENOENT;
 249
 250         for (i = 0; i < adapter->guest->irq_nranges; i++) {
 251                 cur = &adapter->guest->irq_avail[i];
 252                 if (irq >= cur->offset &&
 253                         (irq + len) <= (cur->offset + cur->range)) {
 254                         n = irq - cur->offset;
 255                         bitmap_clear(cur->bitmap, n, len);
 256                         pr_devel("guest: release IRQs %#x->%#x\n",
 257                                 irq, irq + len - 1);
 258                         return 0;
 259                 }
 260         }
 261         return -ENOENT;
 262 }
 263
 264 static int guest_reset(struct cxl *adapter)
 265 {
 266         struct cxl_afu *afu = NULL;
 267         int i, rc;
 268
 269         pr_devel("Adapter reset request\n");
 270         spin_lock(&adapter->afu_list_lock);
 271         for (i = 0; i < adapter->slices; i++) {
 272                 if ((afu = adapter->afu[i])) {
 273                         pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
 274                                         pci_channel_io_frozen);
 275                         cxl_context_detach_all(afu);
 276                 }
 277         }
 278
 279         rc = cxl_h_reset_adapter(adapter->guest->handle);
 280         for (i = 0; i < adapter->slices; i++) {
 281                 if (!rc && (afu = adapter->afu[i])) {
 282                         pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,
 283                                         pci_channel_io_normal);
 284                         pci_error_handlers(afu, CXL_RESUME_EVENT, 0);
 285                 }
 286         }
 287         spin_unlock(&adapter->afu_list_lock);
 288         return rc;
 289 }
 290
 291 static int guest_alloc_one_irq(struct cxl *adapter)
 292 {
 293         int irq;
 294
 295         spin_lock(&adapter->guest->irq_alloc_lock);
 296         if (irq_alloc_range(adapter, 1, &irq))
 297                 irq = -ENOSPC;
 298         spin_unlock(&adapter->guest->irq_alloc_lock);
 299         return irq;
 300 }
 301
 302 static void guest_release_one_irq(struct cxl *adapter, int irq)
 303 {
 304         spin_lock(&adapter->guest->irq_alloc_lock);
 305         irq_free_range(adapter, irq, 1);
 306         spin_unlock(&adapter->guest->irq_alloc_lock);
 307 }
 308
 309 static int guest_alloc_irq_ranges(struct cxl_irq_ranges *irqs,
 310                                 struct cxl *adapter, unsigned int num)
 311 {
 312         int i, try, irq;
 313
 314         memset(irqs, 0, sizeof(struct cxl_irq_ranges));
 315
 316         spin_lock(&adapter->guest->irq_alloc_lock);
 317         for (i = 0; i < CXL_IRQ_RANGES && num; i++) {
 318                 try = num;
 319                 while (try) {
 320                         if (irq_alloc_range(adapter, try, &irq) == 0)
 321                                 break;
 322                         try /= 2;
 323                 }
 324                 if (!try)
 325                         goto error;
 326                 irqs->offset[i] = irq;
 327                 irqs->range[i] = try;
 328                 num -= try;
 329         }
 330         if (num)
 331                 goto error;
 332         spin_unlock(&adapter->guest->irq_alloc_lock);
 333         return 0;
 334
 335 error:
 336         for (i = 0; i < CXL_IRQ_RANGES; i++)
 337                 irq_free_range(adapter, irqs->offset[i], irqs->range[i]);
 338         spin_unlock(&adapter->guest->irq_alloc_lock);
 339         return -ENOSPC;
 340 }
 341
 342 static void guest_release_irq_ranges(struct cxl_irq_ranges *irqs,
 343                                 struct cxl *adapter)
 344 {
 345         int i;
 346
 347         spin_lock(&adapter->guest->irq_alloc_lock);
 348         for (i = 0; i < CXL_IRQ_RANGES; i++)
 349                 irq_free_range(adapter, irqs->offset[i], irqs->range[i]);
 350         spin_unlock(&adapter->guest->irq_alloc_lock);
 351 }
 352
 353 static int guest_register_serr_irq(struct cxl_afu *afu)
 354 {
 355         afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
 356                                       dev_name(&afu->dev));
 357         if (!afu->err_irq_name)
 358                 return -ENOMEM;
 359
 360         if (!(afu->serr_virq = cxl_map_irq(afu->adapter, afu->serr_hwirq,
 361                                  guest_slice_irq_err, afu, afu->err_irq_name))) {
 362                 kfree(afu->err_irq_name);
 363                 afu->err_irq_name = NULL;
 364                 return -ENOMEM;
 365         }
 366
 367         return 0;
 368 }
 369
 370 static void guest_release_serr_irq(struct cxl_afu *afu)
 371 {
 372         cxl_unmap_irq(afu->serr_virq, afu);
 373         cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);
 374         kfree(afu->err_irq_name);
 375 }
 376
 377 static int guest_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
 378 {
 379         return cxl_h_control_faults(ctx->afu->guest->handle, ctx->process_token,
 380                                 tfc >> 32, (psl_reset_mask != 0));
 381 }
 382
 383 static void disable_afu_irqs(struct cxl_context *ctx)
 384 {
 385         irq_hw_number_t hwirq;
 386         unsigned int virq;
 387         int r, i;
 388
 389         pr_devel("Disabling AFU(%d) interrupts\n", ctx->afu->slice);
 390         for (r = 0; r < CXL_IRQ_RANGES; r++) {
 391                 hwirq = ctx->irqs.offset[r];
 392                 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
 393                         virq = irq_find_mapping(NULL, hwirq);
 394                         disable_irq(virq);
 395                 }
 396         }
 397 }
 398
 399 static void enable_afu_irqs(struct cxl_context *ctx)
 400 {
 401         irq_hw_number_t hwirq;
 402         unsigned int virq;
 403         int r, i;
 404
 405         pr_devel("Enabling AFU(%d) interrupts\n", ctx->afu->slice);
 406         for (r = 0; r < CXL_IRQ_RANGES; r++) {
 407                 hwirq = ctx->irqs.offset[r];
 408                 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
 409                         virq = irq_find_mapping(NULL, hwirq);
 410                         enable_irq(virq);
 411                 }
 412         }
 413 }
 414
 415 static int _guest_afu_cr_readXX(int sz, struct cxl_afu *afu, int cr_idx,
 416                         u64 offset, u64 *val)
 417 {
 418         unsigned long cr;
 419         char c;
 420         int rc = 0;
 421
 422         if (afu->crs_len < sz)
 423                 return -ENOENT;
 424
 425         if (unlikely(offset >= afu->crs_len))
 426                 return -ERANGE;
 427
 428         cr = get_zeroed_page(GFP_KERNEL);
 429         if (!cr)
 430                 return -ENOMEM;
 431
 432         rc = cxl_h_get_config(afu->guest->handle, cr_idx, offset,
 433                         virt_to_phys((void *)cr), sz);
 434         if (rc)
 435                 goto err;
 436
 437         switch (sz) {
 438         case 1:
 439                 c = *((char *) cr);
 440                 *val = c;
 441                 break;
 442         case 2:
 443                 *val = in_le16((u16 *)cr);
 444                 break;
 445         case 4:
 446                 *val = in_le32((unsigned *)cr);
 447                 break;
 448         case 8:
 449                 *val = in_le64((u64 *)cr);
 450                 break;
 451         default:
 452                 WARN_ON(1);
 453         }
 454 err:
 455         free_page(cr);
 456         return rc;
 457 }
 458
 459 static int guest_afu_cr_read32(struct cxl_afu *afu, int cr_idx, u64 offset,
 460                         u32 *out)
 461 {
 462         int rc;
 463         u64 val;
 464
 465         rc = _guest_afu_cr_readXX(4, afu, cr_idx, offset, &val);
 466         if (!rc)
 467                 *out = (u32) val;
 468         return rc;
 469 }
 470
 471 static int guest_afu_cr_read16(struct cxl_afu *afu, int cr_idx, u64 offset,
 472                         u16 *out)
 473 {
 474         int rc;
 475         u64 val;
 476
 477         rc = _guest_afu_cr_readXX(2, afu, cr_idx, offset, &val);
 478         if (!rc)
 479                 *out = (u16) val;
 480         return rc;
 481 }
 482
 483 static int guest_afu_cr_read8(struct cxl_afu *afu, int cr_idx, u64 offset,
 484                         u8 *out)
 485 {
 486         int rc;
 487         u64 val;
 488
 489         rc = _guest_afu_cr_readXX(1, afu, cr_idx, offset, &val);
 490         if (!rc)
 491                 *out = (u8) val;
 492         return rc;
 493 }
 494
 495 static int guest_afu_cr_read64(struct cxl_afu *afu, int cr_idx, u64 offset,
 496                         u64 *out)
 497 {
 498         return _guest_afu_cr_readXX(8, afu, cr_idx, offset, out);
 499 }
 500
 501 static int guest_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)
 502 {
 503         /* config record is not writable from guest */
 504         return -EPERM;
 505 }
 506
 507 static int guest_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)
 508 {
 509         /* config record is not writable from guest */
 510         return -EPERM;
 511 }
 512
 513 static int guest_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)
 514 {
 515         /* config record is not writable from guest */
 516         return -EPERM;
 517 }
 518
 519 static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr)
 520 {
 521         struct cxl_process_element_hcall *elem;
 522         struct cxl *adapter = ctx->afu->adapter;
 523         const struct cred *cred;
 524         u32 pid, idx;
 525         int rc, r, i;
 526         u64 mmio_addr, mmio_size;
 527         __be64 flags = 0;
 528
 529         /* Must be 8 byte aligned and cannot cross a 4096 byte boundary */
 530         if (!(elem = (struct cxl_process_element_hcall *)
 531                         get_zeroed_page(GFP_KERNEL)))
 532                 return -ENOMEM;
 533
 534         elem->version = cpu_to_be64(CXL_PROCESS_ELEMENT_VERSION);
 535         if (ctx->kernel) {
 536                 pid = 0;
 537                 flags |= CXL_PE_TRANSLATION_ENABLED;
 538                 flags |= CXL_PE_PRIVILEGED_PROCESS;
 539                 if (mfmsr() & MSR_SF)
 540                         flags |= CXL_PE_64_BIT;
 541         } else {
 542                 pid = current->pid;
 543                 flags |= CXL_PE_PROBLEM_STATE;
 544                 flags |= CXL_PE_TRANSLATION_ENABLED;
 545                 if (!test_tsk_thread_flag(current, TIF_32BIT))
 546                         flags |= CXL_PE_64_BIT;
 547                 cred = get_current_cred();
 548                 if (uid_eq(cred->euid, GLOBAL_ROOT_UID))
 549                         flags |= CXL_PE_PRIVILEGED_PROCESS;
 550                 put_cred(cred);
 551         }
 552         elem->flags         = cpu_to_be64(flags);
 553         elem->common.tid    = cpu_to_be32(0); /* Unused */
 554         elem->common.pid    = cpu_to_be32(pid);
 555         elem->common.csrp   = cpu_to_be64(0); /* disable */
 556         elem->common.u.psl8.aurp0  = cpu_to_be64(0); /* disable */
 557         elem->common.u.psl8.aurp1  = cpu_to_be64(0); /* disable */
 558
 559         cxl_prefault(ctx, wed);
 560
 561         elem->common.u.psl8.sstp0  = cpu_to_be64(ctx->sstp0);
 562         elem->common.u.psl8.sstp1  = cpu_to_be64(ctx->sstp1);
 563
 564         /*
 565          * Ensure we have at least one interrupt allocated to take faults for
 566          * kernel contexts that may not have allocated any AFU IRQs at all:
 567          */
 568         if (ctx->irqs.range[0] == 0) {
 569                 rc = afu_register_irqs(ctx, 0);
 570                 if (rc)
 571                         goto out_free;
 572         }
 573
 574         for (r = 0; r < CXL_IRQ_RANGES; r++) {
 575                 for (i = 0; i < ctx->irqs.range[r]; i++) {
 576                         if (r == 0 && i == 0) {
 577                                 elem->pslVirtualIsn = cpu_to_be32(ctx->irqs.offset[0]);
 578                         } else {
 579                                 idx = ctx->irqs.offset[r] + i - adapter->guest->irq_base_offset;
 580                                 elem->applicationVirtualIsnBitmap[idx / 8] |= 0x80 >> (idx % 8);
 581                         }
 582                 }
 583         }
 584         elem->common.amr = cpu_to_be64(amr);
 585         elem->common.wed = cpu_to_be64(wed);
 586
 587         disable_afu_irqs(ctx);
 588
 589         rc = cxl_h_attach_process(ctx->afu->guest->handle, elem,
 590                                 &ctx->process_token, &mmio_addr, &mmio_size);
 591         if (rc == H_SUCCESS) {
 592                 if (ctx->master || !ctx->afu->pp_psa) {
 593                         ctx->psn_phys = ctx->afu->psn_phys;
 594                         ctx->psn_size = ctx->afu->adapter->ps_size;
 595                 } else {
 596                         ctx->psn_phys = mmio_addr;
 597                         ctx->psn_size = mmio_size;
 598                 }
 599                 if (ctx->afu->pp_psa && mmio_size &&
 600                         ctx->afu->pp_size == 0) {
 601                         /*
 602                          * There's no property in the device tree to read the
 603                          * pp_size. We only find out at the 1st attach.
 604                          * Compared to bare-metal, it is too late and we
 605                          * should really lock here. However, on powerVM,
 606                          * pp_size is really only used to display in /sys.
 607                          * Being discussed with pHyp for their next release.
 608                          */
 609                         ctx->afu->pp_size = mmio_size;
 610                 }
 611                 /* from PAPR: process element is bytes 4-7 of process token */
 612                 ctx->external_pe = ctx->process_token & 0xFFFFFFFF;
 613                 pr_devel("CXL pe=%i is known as %i for pHyp, mmio_size=%#llx",
 614                         ctx->pe, ctx->external_pe, ctx->psn_size);
 615                 ctx->pe_inserted = true;
 616                 enable_afu_irqs(ctx);
 617         }
 618
 619 out_free:
 620         free_page((u64)elem);
 621         return rc;
 622 }
 623
 624 static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr)
 625 {
 626         pr_devel("in %s\n", __func__);
 627
 628         if (ctx->real_mode)
 629                 return -EPERM;
 630
 631         ctx->kernel = kernel;
 632         if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
 633                 return attach_afu_directed(ctx, wed, amr);
 634
 635         /* dedicated mode not supported on FW840 */
 636
 637         return -EINVAL;
 638 }
 639
 640 static int detach_afu_directed(struct cxl_context *ctx)
 641 {
 642         if (!ctx->pe_inserted)
 643                 return 0;
 644         if (cxl_h_detach_process(ctx->afu->guest->handle, ctx->process_token))
 645                 return -1;
 646         return 0;
 647 }
 648
 649 static int guest_detach_process(struct cxl_context *ctx)
 650 {
 651         pr_devel("in %s\n", __func__);
 652         trace_cxl_detach(ctx);
 653
 654         if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
 655                 return -EIO;
 656
 657         if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
 658                 return detach_afu_directed(ctx);
 659
 660         return -EINVAL;
 661 }
 662
 663 static void guest_release_afu(struct device *dev)
 664 {
 665         struct cxl_afu *afu = to_cxl_afu(dev);
 666
 667         pr_devel("%s\n", __func__);
 668
 669         idr_destroy(&afu->contexts_idr);
 670
 671         kfree(afu->guest);
 672         kfree(afu);
 673 }
 674
 675 ssize_t cxl_guest_read_afu_vpd(struct cxl_afu *afu, void *buf, size_t len)
 676 {
 677         return guest_collect_vpd(NULL, afu, buf, len);
 678 }
 679
 680 #define ERR_BUFF_MAX_COPY_SIZE PAGE_SIZE
 681 static ssize_t guest_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
 682                                         loff_t off, size_t count)
 683 {
 684         void *tbuf = NULL;
 685         int rc = 0;
 686
 687         tbuf = (void *) get_zeroed_page(GFP_KERNEL);
 688         if (!tbuf)
 689                 return -ENOMEM;
 690
 691         rc = cxl_h_get_afu_err(afu->guest->handle,
 692                                off & 0x7,
 693                                virt_to_phys(tbuf),
 694                                count);
 695         if (rc)
 696                 goto err;
 697
 698         if (count > ERR_BUFF_MAX_COPY_SIZE)
 699                 count = ERR_BUFF_MAX_COPY_SIZE - (off & 0x7);
 700         memcpy(buf, tbuf, count);
 701 err:
 702         free_page((u64)tbuf);
 703
 704         return rc;
 705 }
 706
 707 static int guest_afu_check_and_enable(struct cxl_afu *afu)
 708 {
 709         return 0;
 710 }
 711
 712 static bool guest_support_attributes(const char *attr_name,
 713                                      enum cxl_attrs type)
 714 {
 715         switch (type) {
 716         case CXL_ADAPTER_ATTRS:
 717                 if ((strcmp(attr_name, "base_image") == 0) ||
 718                         (strcmp(attr_name, "load_image_on_perst") == 0) ||
 719                         (strcmp(attr_name, "perst_reloads_same_image") == 0) ||
 720                         (strcmp(attr_name, "image_loaded") == 0))
 721                         return false;
 722                 break;
 723         case CXL_AFU_MASTER_ATTRS:
 724                 if ((strcmp(attr_name, "pp_mmio_off") == 0))
 725                         return false;
 726                 break;
 727         case CXL_AFU_ATTRS:
 728                 break;
 729         default:
 730                 break;
 731         }
 732
 733         return true;
 734 }
 735
 736 static int activate_afu_directed(struct cxl_afu *afu)
 737 {
 738         int rc;
 739
 740         dev_info(&afu->dev, "Activating AFU(%d) directed mode\n", afu->slice);
 741
 742         afu->current_mode = CXL_MODE_DIRECTED;
 743
 744         afu->num_procs = afu->max_procs_virtualised;
 745
 746         if ((rc = cxl_chardev_m_afu_add(afu)))
 747                 return rc;
 748
 749         if ((rc = cxl_sysfs_afu_m_add(afu)))
 750                 goto err;
 751
 752         if ((rc = cxl_chardev_s_afu_add(afu)))
 753                 goto err1;
 754
 755         return 0;
 756 err1:
 757         cxl_sysfs_afu_m_remove(afu);
 758 err:
 759         cxl_chardev_afu_remove(afu);
 760         return rc;
 761 }
 762
 763 static int guest_afu_activate_mode(struct cxl_afu *afu, int mode)
 764 {
 765         if (!mode)
 766                 return 0;
 767         if (!(mode & afu->modes_supported))
 768                 return -EINVAL;
 769
 770         if (mode == CXL_MODE_DIRECTED)
 771                 return activate_afu_directed(afu);
 772
 773         if (mode == CXL_MODE_DEDICATED)
 774                 dev_err(&afu->dev, "Dedicated mode not supported\n");
 775
 776         return -EINVAL;
 777 }
 778
 779 static int deactivate_afu_directed(struct cxl_afu *afu)
 780 {
 781         dev_info(&afu->dev, "Deactivating AFU(%d) directed mode\n", afu->slice);
 782
 783         afu->current_mode = 0;
 784         afu->num_procs = 0;
 785
 786         cxl_sysfs_afu_m_remove(afu);
 787         cxl_chardev_afu_remove(afu);
 788
 789         cxl_ops->afu_reset(afu);
 790
 791         return 0;
 792 }
 793
 794 static int guest_afu_deactivate_mode(struct cxl_afu *afu, int mode)
 795 {
 796         if (!mode)
 797                 return 0;
 798         if (!(mode & afu->modes_supported))
 799                 return -EINVAL;
 800
 801         if (mode == CXL_MODE_DIRECTED)
 802                 return deactivate_afu_directed(afu);
 803         return 0;
 804 }
 805
 806 static int guest_afu_reset(struct cxl_afu *afu)
 807 {
 808         pr_devel("AFU(%d) reset request\n", afu->slice);
 809         return cxl_h_reset_afu(afu->guest->handle);
 810 }
 811
 812 static int guest_map_slice_regs(struct cxl_afu *afu)
 813 {
 814         if (!(afu->p2n_mmio = ioremap(afu->guest->p2n_phys, afu->guest->p2n_size))) {
 815                 dev_err(&afu->dev, "Error mapping AFU(%d) MMIO regions\n",
 816                         afu->slice);
 817                 return -ENOMEM;
 818         }
 819         return 0;
 820 }
 821
 822 static void guest_unmap_slice_regs(struct cxl_afu *afu)
 823 {
 824         if (afu->p2n_mmio)
 825                 iounmap(afu->p2n_mmio);
 826 }
 827
 828 static int afu_update_state(struct cxl_afu *afu)
 829 {
 830         int rc, cur_state;
 831
 832         rc = afu_read_error_state(afu, &cur_state);
 833         if (rc)
 834                 return rc;
 835
 836         if (afu->guest->previous_state == cur_state)
 837                 return 0;
 838
 839         pr_devel("AFU(%d) update state to %#x\n", afu->slice, cur_state);
 840
 841         switch (cur_state) {
 842         case H_STATE_NORMAL:
 843                 afu->guest->previous_state = cur_state;
 844                 break;
 845
 846         case H_STATE_DISABLE:
 847                 pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
 848                                 pci_channel_io_frozen);
 849
 850                 cxl_context_detach_all(afu);
 851                 if ((rc = cxl_ops->afu_reset(afu)))
 852                         pr_devel("reset hcall failed %d\n", rc);
 853
 854                 rc = afu_read_error_state(afu, &cur_state);
 855                 if (!rc && cur_state == H_STATE_NORMAL) {
 856                         pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,
 857                                         pci_channel_io_normal);
 858                         pci_error_handlers(afu, CXL_RESUME_EVENT, 0);
 859                 }
 860                 afu->guest->previous_state = 0;
 861                 break;
 862
 863         case H_STATE_TEMP_UNAVAILABLE:
 864                 afu->guest->previous_state = cur_state;
 865                 break;
 866
 867         case H_STATE_PERM_UNAVAILABLE:
 868                 dev_err(&afu->dev, "AFU is in permanent error state\n");
 869                 pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
 870                                 pci_channel_io_perm_failure);
 871                 afu->guest->previous_state = cur_state;
 872                 break;
 873
 874         default:
 875                 pr_err("Unexpected AFU(%d) error state: %#x\n",
 876                        afu->slice, cur_state);
 877                 return -EINVAL;
 878         }
 879
 880         return rc;
 881 }
 882
 883 static void afu_handle_errstate(struct work_struct *work)
 884 {
 885         struct cxl_afu_guest *afu_guest =
 886                 container_of(to_delayed_work(work), struct cxl_afu_guest, work_err);
 887
 888         if (!afu_update_state(afu_guest->parent) &&
 889             afu_guest->previous_state == H_STATE_PERM_UNAVAILABLE)
 890                 return;
 891
 892         if (afu_guest->handle_err)
 893                 schedule_delayed_work(&afu_guest->work_err,
 894                                       msecs_to_jiffies(3000));
 895 }
 896
 897 static bool guest_link_ok(struct cxl *cxl, struct cxl_afu *afu)
 898 {
 899         int state;
 900
 901         if (afu && (!afu_read_error_state(afu, &state))) {
 902                 if (state == H_STATE_NORMAL)
 903                         return true;
 904         }
 905
 906         return false;
 907 }
 908
 909 static int afu_properties_look_ok(struct cxl_afu *afu)
 910 {
 911         if (afu->pp_irqs < 0) {
 912                 dev_err(&afu->dev, "Unexpected per-process minimum interrupt value\n");
 913                 return -EINVAL;
 914         }
 915
 916         if (afu->max_procs_virtualised < 1) {
 917                 dev_err(&afu->dev, "Unexpected max number of processes virtualised value\n");
 918                 return -EINVAL;
 919         }
 920
 921         if (afu->crs_len < 0) {
 922                 dev_err(&afu->dev, "Unexpected configuration record size value\n");
 923                 return -EINVAL;
 924         }
 925
 926         return 0;
 927 }
 928
 929 int cxl_guest_init_afu(struct cxl *adapter, int slice, struct device_node *afu_np)
 930 {
 931         struct cxl_afu *afu;
 932         bool free = true;
 933         int rc;
 934
 935         pr_devel("in %s - AFU(%d)\n", __func__, slice);
 936         if (!(afu = cxl_alloc_afu(adapter, slice)))
 937                 return -ENOMEM;
 938
 939         if (!(afu->guest = kzalloc(sizeof(struct cxl_afu_guest), GFP_KERNEL))) {
 940                 kfree(afu);
 941                 return -ENOMEM;
 942         }
 943
 944         if ((rc = dev_set_name(&afu->dev, "afu%i.%i",
 945                                           adapter->adapter_num,
 946                                           slice)))
 947                 goto err1;
 948
 949         adapter->slices++;
 950
 951         if ((rc = cxl_of_read_afu_handle(afu, afu_np)))
 952                 goto err1;
 953
 954         if ((rc = cxl_ops->afu_reset(afu)))
 955                 goto err1;
 956
 957         if ((rc = cxl_of_read_afu_properties(afu, afu_np)))
 958                 goto err1;
 959
 960         if ((rc = afu_properties_look_ok(afu)))
 961                 goto err1;
 962
 963         if ((rc = guest_map_slice_regs(afu)))
 964                 goto err1;
 965
 966         if ((rc = guest_register_serr_irq(afu)))
 967                 goto err2;
 968
 969         /*
 970          * After we call this function we must not free the afu directly, even
 971          * if it returns an error!
 972          */
 973         if ((rc = cxl_register_afu(afu)))
 974                 goto err_put1;
 975
 976         if ((rc = cxl_sysfs_afu_add(afu)))
 977                 goto err_put1;
 978
 979         /*
 980          * pHyp doesn't expose the programming models supported by the
 981          * AFU. pHyp currently only supports directed mode. If it adds
 982          * dedicated mode later, this version of cxl has no way to
 983          * detect it. So we'll initialize the driver, but the first
 984          * attach will fail.
 985          * Being discussed with pHyp to do better (likely new property)
 986          */
 987         if (afu->max_procs_virtualised == 1)
 988                 afu->modes_supported = CXL_MODE_DEDICATED;
 989         else
 990                 afu->modes_supported = CXL_MODE_DIRECTED;
 991
 992         if ((rc = cxl_afu_select_best_mode(afu)))
 993                 goto err_put2;
 994
 995         adapter->afu[afu->slice] = afu;
 996
 997         afu->enabled = true;
 998
 999         /*
1000          * wake up the cpu periodically to check the state
1001          * of the AFU using "afu" stored in the guest structure.
1002          */
1003         afu->guest->parent = afu;
1004         afu->guest->handle_err = true;
1005         INIT_DELAYED_WORK(&afu->guest->work_err, afu_handle_errstate);
1006         schedule_delayed_work(&afu->guest->work_err, msecs_to_jiffies(1000));
1007
1008         if ((rc = cxl_pci_vphb_add(afu)))
1009                 dev_info(&afu->dev, "Can't register vPHB\n");
1010
1011         return 0;
1012
1013 err_put2:
1014         cxl_sysfs_afu_remove(afu);
1015 err_put1:
1016         device_unregister(&afu->dev);
1017         free = false;
1018         guest_release_serr_irq(afu);
1019 err2:
1020         guest_unmap_slice_regs(afu);
1021 err1:
1022         if (free) {
1023                 kfree(afu->guest);
1024                 kfree(afu);
1025         }
1026         return rc;
1027 }
1028
1029 void cxl_guest_remove_afu(struct cxl_afu *afu)
1030 {
1031         if (!afu)
1032                 return;
1033
1034         /* flush and stop pending job */
1035         afu->guest->handle_err = false;
1036         flush_delayed_work(&afu->guest->work_err);
1037
1038         cxl_pci_vphb_remove(afu);
1039         cxl_sysfs_afu_remove(afu);
1040
1041         spin_lock(&afu->adapter->afu_list_lock);
1042         afu->adapter->afu[afu->slice] = NULL;
1043         spin_unlock(&afu->adapter->afu_list_lock);
1044
1045         cxl_context_detach_all(afu);
1046         cxl_ops->afu_deactivate_mode(afu, afu->current_mode);
1047         guest_release_serr_irq(afu);
1048         guest_unmap_slice_regs(afu);
1049
1050         device_unregister(&afu->dev);
1051 }
1052
1053 static void free_adapter(struct cxl *adapter)
1054 {
1055         struct irq_avail *cur;
1056         int i;
1057
1058         if (adapter->guest) {
1059                 if (adapter->guest->irq_avail) {
1060                         for (i = 0; i < adapter->guest->irq_nranges; i++) {
1061                                 cur = &adapter->guest->irq_avail[i];
1062                                 kfree(cur->bitmap);
1063                         }
1064                         kfree(adapter->guest->irq_avail);
1065                 }
1066                 kfree(adapter->guest->status);
1067                 kfree(adapter->guest);
1068         }
1069         cxl_remove_adapter_nr(adapter);
1070         kfree(adapter);
1071 }
1072
1073 static int properties_look_ok(struct cxl *adapter)
1074 {
1075         /* The absence of this property means that the operational
1076          * status is unknown or okay
1077          */
1078         if (strlen(adapter->guest->status) &&
1079             strcmp(adapter->guest->status, "okay")) {
1080                 pr_err("ABORTING:Bad operational status of the device\n");
1081                 return -EINVAL;
1082         }
1083
1084         return 0;
1085 }
1086
1087 ssize_t cxl_guest_read_adapter_vpd(struct cxl *adapter, void *buf, size_t len)
1088 {
1089         return guest_collect_vpd(adapter, NULL, buf, len);
1090 }
1091
1092 void cxl_guest_remove_adapter(struct cxl *adapter)
1093 {
1094         pr_devel("in %s\n", __func__);
1095
1096         cxl_sysfs_adapter_remove(adapter);
1097
1098         cxl_guest_remove_chardev(adapter);
1099         device_unregister(&adapter->dev);
1100 }
1101
1102 static void release_adapter(struct device *dev)
1103 {
1104         free_adapter(to_cxl_adapter(dev));
1105 }
1106
1107 struct cxl *cxl_guest_init_adapter(struct device_node *np, struct platform_device *pdev)
1108 {
1109         struct cxl *adapter;
1110         bool free = true;
1111         int rc;
1112
1113         if (!(adapter = cxl_alloc_adapter()))
1114                 return ERR_PTR(-ENOMEM);
1115
1116         if (!(adapter->guest = kzalloc(sizeof(struct cxl_guest), GFP_KERNEL))) {
1117                 free_adapter(adapter);
1118                 return ERR_PTR(-ENOMEM);
1119         }
1120
1121         adapter->slices = 0;
1122         adapter->guest->pdev = pdev;
1123         adapter->dev.parent = &pdev->dev;
1124         adapter->dev.release = release_adapter;
1125         dev_set_drvdata(&pdev->dev, adapter);
1126
1127         /*
1128          * Hypervisor controls PSL timebase initialization (p1 register).
1129          * On FW840, PSL is initialized.
1130          */
1131         adapter->psl_timebase_synced = true;
1132
1133         if ((rc = cxl_of_read_adapter_handle(adapter, np)))
1134                 goto err1;
1135
1136         if ((rc = cxl_of_read_adapter_properties(adapter, np)))
1137                 goto err1;
1138
1139         if ((rc = properties_look_ok(adapter)))
1140                 goto err1;
1141
1142         if ((rc = cxl_guest_add_chardev(adapter)))
1143                 goto err1;
1144
1145         /*
1146          * After we call this function we must not free the adapter directly,
1147          * even if it returns an error!
1148          */
1149         if ((rc = cxl_register_adapter(adapter)))
1150                 goto err_put1;
1151
1152         if ((rc = cxl_sysfs_adapter_add(adapter)))
1153                 goto err_put1;
1154
1155         /* release the context lock as the adapter is configured */
1156         cxl_adapter_context_unlock(adapter);
1157
1158         return adapter;
1159
1160 err_put1:
1161         device_unregister(&adapter->dev);
1162         free = false;
1163         cxl_guest_remove_chardev(adapter);
1164 err1:
1165         if (free)
1166                 free_adapter(adapter);
1167         return ERR_PTR(rc);
1168 }
1169
1170 void cxl_guest_reload_module(struct cxl *adapter)
1171 {
1172         struct platform_device *pdev;
1173
1174         pdev = adapter->guest->pdev;
1175         cxl_guest_remove_adapter(adapter);
1176
1177         cxl_of_probe(pdev);
1178 }
1179
1180 const struct cxl_backend_ops cxl_guest_ops = {
1181         .module = THIS_MODULE,
1182         .adapter_reset = guest_reset,
1183         .alloc_one_irq = guest_alloc_one_irq,
1184         .release_one_irq = guest_release_one_irq,
1185         .alloc_irq_ranges = guest_alloc_irq_ranges,
1186         .release_irq_ranges = guest_release_irq_ranges,
1187         .setup_irq = NULL,
1188         .handle_psl_slice_error = guest_handle_psl_slice_error,
1189         .psl_interrupt = guest_psl_irq,
1190         .ack_irq = guest_ack_irq,
1191         .attach_process = guest_attach_process,
1192         .detach_process = guest_detach_process,
1193         .update_ivtes = NULL,
1194         .support_attributes = guest_support_attributes,
1195         .link_ok = guest_link_ok,
1196         .release_afu = guest_release_afu,
1197         .afu_read_err_buffer = guest_afu_read_err_buffer,
1198         .afu_check_and_enable = guest_afu_check_and_enable,
1199         .afu_activate_mode = guest_afu_activate_mode,
1200         .afu_deactivate_mode = guest_afu_deactivate_mode,
1201         .afu_reset = guest_afu_reset,
1202         .afu_cr_read8 = guest_afu_cr_read8,
1203         .afu_cr_read16 = guest_afu_cr_read16,
1204         .afu_cr_read32 = guest_afu_cr_read32,
1205         .afu_cr_read64 = guest_afu_cr_read64,
1206         .afu_cr_write8 = guest_afu_cr_write8,
1207         .afu_cr_write16 = guest_afu_cr_write16,
1208         .afu_cr_write32 = guest_afu_cr_write32,
1209         .read_adapter_vpd = cxl_guest_read_adapter_vpd,
1210 };