drivers/firmware/arm_sdei.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (C) 2017 Arm Ltd.
   3 #define pr_fmt(fmt) "sdei: " fmt
   4
   5 #include <linux/acpi.h>
   6 #include <linux/arm_sdei.h>
   7 #include <linux/arm-smccc.h>
   8 #include <linux/atomic.h>
   9 #include <linux/bitops.h>
  10 #include <linux/compiler.h>
  11 #include <linux/cpuhotplug.h>
  12 #include <linux/cpu.h>
  13 #include <linux/cpu_pm.h>
  14 #include <linux/errno.h>
  15 #include <linux/hardirq.h>
  16 #include <linux/kernel.h>
  17 #include <linux/kprobes.h>
  18 #include <linux/kvm_host.h>
  19 #include <linux/list.h>
  20 #include <linux/mutex.h>
  21 #include <linux/notifier.h>
  22 #include <linux/of.h>
  23 #include <linux/of_platform.h>
  24 #include <linux/percpu.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/pm.h>
  27 #include <linux/ptrace.h>
  28 #include <linux/preempt.h>
  29 #include <linux/reboot.h>
  30 #include <linux/slab.h>
  31 #include <linux/smp.h>
  32 #include <linux/spinlock.h>
  33 #include <linux/uaccess.h>
  34
  35 /*
  36  * The call to use to reach the firmware.
  37  */
  38 static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
  39                       unsigned long arg0, unsigned long arg1,
  40                       unsigned long arg2, unsigned long arg3,
  41                       unsigned long arg4, struct arm_smccc_res *res);
  42
  43 /* entry point from firmware to arch asm code */
  44 static unsigned long sdei_entry_point;
  45
  46 struct sdei_event {
  47         /* These three are protected by the sdei_list_lock */
  48         struct list_head        list;
  49         bool                    reregister;
  50         bool                    reenable;
  51
  52         u32                     event_num;
  53         u8                      type;
  54         u8                      priority;
  55
  56         /* This pointer is handed to firmware as the event argument. */
  57         union {
  58                 /* Shared events */
  59                 struct sdei_registered_event *registered;
  60
  61                 /* CPU private events */
  62                 struct sdei_registered_event __percpu *private_registered;
  63         };
  64 };
  65
  66 /* Take the mutex for any API call or modification. Take the mutex first. */
  67 static DEFINE_MUTEX(sdei_events_lock);
  68
  69 /* and then hold this when modifying the list */
  70 static DEFINE_SPINLOCK(sdei_list_lock);
  71 static LIST_HEAD(sdei_list);
  72
  73 /* Private events are registered/enabled via IPI passing one of these */
  74 struct sdei_crosscall_args {
  75         struct sdei_event *event;
  76         atomic_t errors;
  77         int first_error;
  78 };
  79
  80 #define CROSSCALL_INIT(arg, event)      (arg.event = event, \
  81                                          arg.first_error = 0, \
  82                                          atomic_set(&arg.errors, 0))
  83
  84 static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
  85 {
  86         struct sdei_crosscall_args arg;
  87
  88         CROSSCALL_INIT(arg, event);
  89         on_each_cpu(fn, &arg, true);
  90
  91         return arg.first_error;
  92 }
  93
  94 static inline void
  95 sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
  96 {
  97         if (err && (atomic_inc_return(&arg->errors) == 1))
  98                 arg->first_error = err;
  99 }
 100
 101 static int sdei_to_linux_errno(unsigned long sdei_err)
 102 {
 103         switch (sdei_err) {
 104         case SDEI_NOT_SUPPORTED:
 105                 return -EOPNOTSUPP;
 106         case SDEI_INVALID_PARAMETERS:
 107                 return -EINVAL;
 108         case SDEI_DENIED:
 109                 return -EPERM;
 110         case SDEI_PENDING:
 111                 return -EINPROGRESS;
 112         case SDEI_OUT_OF_RESOURCE:
 113                 return -ENOMEM;
 114         }
 115
 116         /* Not an error value ... */
 117         return sdei_err;
 118 }
 119
 120 /*
 121  * If x0 is any of these values, then the call failed, use sdei_to_linux_errno()
 122  * to translate.
 123  */
 124 static int sdei_is_err(struct arm_smccc_res *res)
 125 {
 126         switch (res->a0) {
 127         case SDEI_NOT_SUPPORTED:
 128         case SDEI_INVALID_PARAMETERS:
 129         case SDEI_DENIED:
 130         case SDEI_PENDING:
 131         case SDEI_OUT_OF_RESOURCE:
 132                 return true;
 133         }
 134
 135         return false;
 136 }
 137
 138 static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
 139                           unsigned long arg1, unsigned long arg2,
 140                           unsigned long arg3, unsigned long arg4,
 141                           u64 *result)
 142 {
 143         int err = 0;
 144         struct arm_smccc_res res;
 145
 146         if (sdei_firmware_call) {
 147                 sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
 148                                    &res);
 149                 if (sdei_is_err(&res))
 150                         err = sdei_to_linux_errno(res.a0);
 151         } else {
 152                 /*
 153                  * !sdei_firmware_call means we failed to probe or called
 154                  * sdei_mark_interface_broken(). -EIO is not an error returned
 155                  * by sdei_to_linux_errno() and is used to suppress messages
 156                  * from this driver.
 157                  */
 158                 err = -EIO;
 159                 res.a0 = SDEI_NOT_SUPPORTED;
 160         }
 161
 162         if (result)
 163                 *result = res.a0;
 164
 165         return err;
 166 }
 167
 168 static struct sdei_event *sdei_event_find(u32 event_num)
 169 {
 170         struct sdei_event *e, *found = NULL;
 171
 172         lockdep_assert_held(&sdei_events_lock);
 173
 174         spin_lock(&sdei_list_lock);
 175         list_for_each_entry(e, &sdei_list, list) {
 176                 if (e->event_num == event_num) {
 177                         found = e;
 178                         break;
 179                 }
 180         }
 181         spin_unlock(&sdei_list_lock);
 182
 183         return found;
 184 }
 185
 186 int sdei_api_event_context(u32 query, u64 *result)
 187 {
 188         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_CONTEXT, query, 0, 0, 0, 0,
 189                               result);
 190 }
 191 NOKPROBE_SYMBOL(sdei_api_event_context);
 192
 193 static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
 194 {
 195         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
 196                               0, 0, result);
 197 }
 198
 199 static struct sdei_event *sdei_event_create(u32 event_num,
 200                                             sdei_event_callback *cb,
 201                                             void *cb_arg)
 202 {
 203         int err;
 204         u64 result;
 205         struct sdei_event *event;
 206         struct sdei_registered_event *reg;
 207
 208         lockdep_assert_held(&sdei_events_lock);
 209
 210         event = kzalloc(sizeof(*event), GFP_KERNEL);
 211         if (!event)
 212                 return ERR_PTR(-ENOMEM);
 213
 214         INIT_LIST_HEAD(&event->list);
 215         event->event_num = event_num;
 216
 217         err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
 218                                       &result);
 219         if (err) {
 220                 kfree(event);
 221                 return ERR_PTR(err);
 222         }
 223         event->priority = result;
 224
 225         err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
 226                                       &result);
 227         if (err) {
 228                 kfree(event);
 229                 return ERR_PTR(err);
 230         }
 231         event->type = result;
 232
 233         if (event->type == SDEI_EVENT_TYPE_SHARED) {
 234                 reg = kzalloc(sizeof(*reg), GFP_KERNEL);
 235                 if (!reg) {
 236                         kfree(event);
 237                         return ERR_PTR(-ENOMEM);
 238                 }
 239
 240                 reg->event_num = event_num;
 241                 reg->priority = event->priority;
 242
 243                 reg->callback = cb;
 244                 reg->callback_arg = cb_arg;
 245                 event->registered = reg;
 246         } else {
 247                 int cpu;
 248                 struct sdei_registered_event __percpu *regs;
 249
 250                 regs = alloc_percpu(struct sdei_registered_event);
 251                 if (!regs) {
 252                         kfree(event);
 253                         return ERR_PTR(-ENOMEM);
 254                 }
 255
 256                 for_each_possible_cpu(cpu) {
 257                         reg = per_cpu_ptr(regs, cpu);
 258
 259                         reg->event_num = event->event_num;
 260                         reg->priority = event->priority;
 261                         reg->callback = cb;
 262                         reg->callback_arg = cb_arg;
 263                 }
 264
 265                 event->private_registered = regs;
 266         }
 267
 268         if (sdei_event_find(event_num)) {
 269                 kfree(event->registered);
 270                 kfree(event);
 271                 event = ERR_PTR(-EBUSY);
 272         } else {
 273                 spin_lock(&sdei_list_lock);
 274                 list_add(&event->list, &sdei_list);
 275                 spin_unlock(&sdei_list_lock);
 276         }
 277
 278         return event;
 279 }
 280
 281 static void sdei_event_destroy(struct sdei_event *event)
 282 {
 283         lockdep_assert_held(&sdei_events_lock);
 284
 285         spin_lock(&sdei_list_lock);
 286         list_del(&event->list);
 287         spin_unlock(&sdei_list_lock);
 288
 289         if (event->type == SDEI_EVENT_TYPE_SHARED)
 290                 kfree(event->registered);
 291         else
 292                 free_percpu(event->private_registered);
 293
 294         kfree(event);
 295 }
 296
 297 static int sdei_api_get_version(u64 *version)
 298 {
 299         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
 300 }
 301
 302 int sdei_mask_local_cpu(void)
 303 {
 304         int err;
 305
 306         WARN_ON_ONCE(preemptible());
 307
 308         err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
 309         if (err && err != -EIO) {
 310                 pr_warn_once("failed to mask CPU[%u]: %d\n",
 311                               smp_processor_id(), err);
 312                 return err;
 313         }
 314
 315         return 0;
 316 }
 317
 318 static void _ipi_mask_cpu(void *ignored)
 319 {
 320         sdei_mask_local_cpu();
 321 }
 322
 323 int sdei_unmask_local_cpu(void)
 324 {
 325         int err;
 326
 327         WARN_ON_ONCE(preemptible());
 328
 329         err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
 330         if (err && err != -EIO) {
 331                 pr_warn_once("failed to unmask CPU[%u]: %d\n",
 332                              smp_processor_id(), err);
 333                 return err;
 334         }
 335
 336         return 0;
 337 }
 338
 339 static void _ipi_unmask_cpu(void *ignored)
 340 {
 341         sdei_unmask_local_cpu();
 342 }
 343
 344 static void _ipi_private_reset(void *ignored)
 345 {
 346         int err;
 347
 348         err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
 349                              NULL);
 350         if (err && err != -EIO)
 351                 pr_warn_once("failed to reset CPU[%u]: %d\n",
 352                              smp_processor_id(), err);
 353 }
 354
 355 static int sdei_api_shared_reset(void)
 356 {
 357         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_SHARED_RESET, 0, 0, 0, 0, 0,
 358                               NULL);
 359 }
 360
 361 static void sdei_mark_interface_broken(void)
 362 {
 363         pr_err("disabling SDEI firmware interface\n");
 364         on_each_cpu(&_ipi_mask_cpu, NULL, true);
 365         sdei_firmware_call = NULL;
 366 }
 367
 368 static int sdei_platform_reset(void)
 369 {
 370         int err;
 371
 372         on_each_cpu(&_ipi_private_reset, NULL, true);
 373         err = sdei_api_shared_reset();
 374         if (err) {
 375                 pr_err("Failed to reset platform: %d\n", err);
 376                 sdei_mark_interface_broken();
 377         }
 378
 379         return err;
 380 }
 381
 382 static int sdei_api_event_enable(u32 event_num)
 383 {
 384         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
 385                               0, NULL);
 386 }
 387
 388 /* Called directly by the hotplug callbacks */
 389 static void _local_event_enable(void *data)
 390 {
 391         int err;
 392         struct sdei_crosscall_args *arg = data;
 393
 394         WARN_ON_ONCE(preemptible());
 395
 396         err = sdei_api_event_enable(arg->event->event_num);
 397
 398         sdei_cross_call_return(arg, err);
 399 }
 400
 401 int sdei_event_enable(u32 event_num)
 402 {
 403         int err = -EINVAL;
 404         struct sdei_event *event;
 405
 406         mutex_lock(&sdei_events_lock);
 407         event = sdei_event_find(event_num);
 408         if (!event) {
 409                 mutex_unlock(&sdei_events_lock);
 410                 return -ENOENT;
 411         }
 412
 413
 414         cpus_read_lock();
 415         if (event->type == SDEI_EVENT_TYPE_SHARED)
 416                 err = sdei_api_event_enable(event->event_num);
 417         else
 418                 err = sdei_do_cross_call(_local_event_enable, event);
 419
 420         if (!err) {
 421                 spin_lock(&sdei_list_lock);
 422                 event->reenable = true;
 423                 spin_unlock(&sdei_list_lock);
 424         }
 425         cpus_read_unlock();
 426         mutex_unlock(&sdei_events_lock);
 427
 428         return err;
 429 }
 430 EXPORT_SYMBOL(sdei_event_enable);
 431
 432 static int sdei_api_event_disable(u32 event_num)
 433 {
 434         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
 435                               0, 0, NULL);
 436 }
 437
 438 static void _ipi_event_disable(void *data)
 439 {
 440         int err;
 441         struct sdei_crosscall_args *arg = data;
 442
 443         err = sdei_api_event_disable(arg->event->event_num);
 444
 445         sdei_cross_call_return(arg, err);
 446 }
 447
 448 int sdei_event_disable(u32 event_num)
 449 {
 450         int err = -EINVAL;
 451         struct sdei_event *event;
 452
 453         mutex_lock(&sdei_events_lock);
 454         event = sdei_event_find(event_num);
 455         if (!event) {
 456                 mutex_unlock(&sdei_events_lock);
 457                 return -ENOENT;
 458         }
 459
 460         spin_lock(&sdei_list_lock);
 461         event->reenable = false;
 462         spin_unlock(&sdei_list_lock);
 463
 464         if (event->type == SDEI_EVENT_TYPE_SHARED)
 465                 err = sdei_api_event_disable(event->event_num);
 466         else
 467                 err = sdei_do_cross_call(_ipi_event_disable, event);
 468         mutex_unlock(&sdei_events_lock);
 469
 470         return err;
 471 }
 472 EXPORT_SYMBOL(sdei_event_disable);
 473
 474 static int sdei_api_event_unregister(u32 event_num)
 475 {
 476         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0,
 477                               0, 0, 0, NULL);
 478 }
 479
 480 /* Called directly by the hotplug callbacks */
 481 static void _local_event_unregister(void *data)
 482 {
 483         int err;
 484         struct sdei_crosscall_args *arg = data;
 485
 486         WARN_ON_ONCE(preemptible());
 487
 488         err = sdei_api_event_unregister(arg->event->event_num);
 489
 490         sdei_cross_call_return(arg, err);
 491 }
 492
 493 static int _sdei_event_unregister(struct sdei_event *event)
 494 {
 495         lockdep_assert_held(&sdei_events_lock);
 496
 497         if (event->type == SDEI_EVENT_TYPE_SHARED)
 498                 return sdei_api_event_unregister(event->event_num);
 499
 500         return sdei_do_cross_call(_local_event_unregister, event);
 501 }
 502
 503 int sdei_event_unregister(u32 event_num)
 504 {
 505         int err;
 506         struct sdei_event *event;
 507
 508         WARN_ON(in_nmi());
 509
 510         mutex_lock(&sdei_events_lock);
 511         event = sdei_event_find(event_num);
 512         do {
 513                 if (!event) {
 514                         pr_warn("Event %u not registered\n", event_num);
 515                         err = -ENOENT;
 516                         break;
 517                 }
 518
 519                 spin_lock(&sdei_list_lock);
 520                 event->reregister = false;
 521                 event->reenable = false;
 522                 spin_unlock(&sdei_list_lock);
 523
 524                 err = _sdei_event_unregister(event);
 525                 if (err)
 526                         break;
 527
 528                 sdei_event_destroy(event);
 529         } while (0);
 530         mutex_unlock(&sdei_events_lock);
 531
 532         return err;
 533 }
 534 EXPORT_SYMBOL(sdei_event_unregister);
 535
 536 /*
 537  * unregister events, but don't destroy them as they are re-registered by
 538  * sdei_reregister_shared().
 539  */
 540 static int sdei_unregister_shared(void)
 541 {
 542         int err = 0;
 543         struct sdei_event *event;
 544
 545         mutex_lock(&sdei_events_lock);
 546         spin_lock(&sdei_list_lock);
 547         list_for_each_entry(event, &sdei_list, list) {
 548                 if (event->type != SDEI_EVENT_TYPE_SHARED)
 549                         continue;
 550
 551                 err = _sdei_event_unregister(event);
 552                 if (err)
 553                         break;
 554         }
 555         spin_unlock(&sdei_list_lock);
 556         mutex_unlock(&sdei_events_lock);
 557
 558         return err;
 559 }
 560
 561 static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
 562                                    void *arg, u64 flags, u64 affinity)
 563 {
 564         return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_REGISTER, event_num,
 565                               (unsigned long)entry_point, (unsigned long)arg,
 566                               flags, affinity, NULL);
 567 }
 568
 569 /* Called directly by the hotplug callbacks */
 570 static void _local_event_register(void *data)
 571 {
 572         int err;
 573         struct sdei_registered_event *reg;
 574         struct sdei_crosscall_args *arg = data;
 575
 576         WARN_ON(preemptible());
 577
 578         reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
 579         err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
 580                                       reg, 0, 0);
 581
 582         sdei_cross_call_return(arg, err);
 583 }
 584
 585 static int _sdei_event_register(struct sdei_event *event)
 586 {
 587         int err;
 588
 589         lockdep_assert_held(&sdei_events_lock);
 590
 591         if (event->type == SDEI_EVENT_TYPE_SHARED)
 592                 return sdei_api_event_register(event->event_num,
 593                                                sdei_entry_point,
 594                                                event->registered,
 595                                                SDEI_EVENT_REGISTER_RM_ANY, 0);
 596
 597         err = sdei_do_cross_call(_local_event_register, event);
 598         if (err)
 599                 sdei_do_cross_call(_local_event_unregister, event);
 600
 601         return err;
 602 }
 603
 604 int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
 605 {
 606         int err;
 607         struct sdei_event *event;
 608
 609         WARN_ON(in_nmi());
 610
 611         mutex_lock(&sdei_events_lock);
 612         do {
 613                 if (sdei_event_find(event_num)) {
 614                         pr_warn("Event %u already registered\n", event_num);
 615                         err = -EBUSY;
 616                         break;
 617                 }
 618
 619                 event = sdei_event_create(event_num, cb, arg);
 620                 if (IS_ERR(event)) {
 621                         err = PTR_ERR(event);
 622                         pr_warn("Failed to create event %u: %d\n", event_num,
 623                                 err);
 624                         break;
 625                 }
 626
 627                 cpus_read_lock();
 628                 err = _sdei_event_register(event);
 629                 if (err) {
 630                         sdei_event_destroy(event);
 631                         pr_warn("Failed to register event %u: %d\n", event_num,
 632                                 err);
 633                 } else {
 634                         spin_lock(&sdei_list_lock);
 635                         event->reregister = true;
 636                         spin_unlock(&sdei_list_lock);
 637                 }
 638                 cpus_read_unlock();
 639         } while (0);
 640         mutex_unlock(&sdei_events_lock);
 641
 642         return err;
 643 }
 644 EXPORT_SYMBOL(sdei_event_register);
 645
 646 static int sdei_reregister_event(struct sdei_event *event)
 647 {
 648         int err;
 649
 650         lockdep_assert_held(&sdei_events_lock);
 651
 652         err = _sdei_event_register(event);
 653         if (err) {
 654                 pr_err("Failed to re-register event %u\n", event->event_num);
 655                 sdei_event_destroy(event);
 656                 return err;
 657         }
 658
 659         if (event->reenable) {
 660                 if (event->type == SDEI_EVENT_TYPE_SHARED)
 661                         err = sdei_api_event_enable(event->event_num);
 662                 else
 663                         err = sdei_do_cross_call(_local_event_enable, event);
 664         }
 665
 666         if (err)
 667                 pr_err("Failed to re-enable event %u\n", event->event_num);
 668
 669         return err;
 670 }
 671
 672 static int sdei_reregister_shared(void)
 673 {
 674         int err = 0;
 675         struct sdei_event *event;
 676
 677         mutex_lock(&sdei_events_lock);
 678         spin_lock(&sdei_list_lock);
 679         list_for_each_entry(event, &sdei_list, list) {
 680                 if (event->type != SDEI_EVENT_TYPE_SHARED)
 681                         continue;
 682
 683                 if (event->reregister) {
 684                         err = sdei_reregister_event(event);
 685                         if (err)
 686                                 break;
 687                 }
 688         }
 689         spin_unlock(&sdei_list_lock);
 690         mutex_unlock(&sdei_events_lock);
 691
 692         return err;
 693 }
 694
 695 static int sdei_cpuhp_down(unsigned int cpu)
 696 {
 697         struct sdei_event *event;
 698         struct sdei_crosscall_args arg;
 699
 700         /* un-register private events */
 701         spin_lock(&sdei_list_lock);
 702         list_for_each_entry(event, &sdei_list, list) {
 703                 if (event->type == SDEI_EVENT_TYPE_SHARED)
 704                         continue;
 705
 706                 CROSSCALL_INIT(arg, event);
 707                 /* call the cross-call function locally... */
 708                 _local_event_unregister(&arg);
 709                 if (arg.first_error)
 710                         pr_err("Failed to unregister event %u: %d\n",
 711                                event->event_num, arg.first_error);
 712         }
 713         spin_unlock(&sdei_list_lock);
 714
 715         return sdei_mask_local_cpu();
 716 }
 717
 718 static int sdei_cpuhp_up(unsigned int cpu)
 719 {
 720         struct sdei_event *event;
 721         struct sdei_crosscall_args arg;
 722
 723         /* re-register/enable private events */
 724         spin_lock(&sdei_list_lock);
 725         list_for_each_entry(event, &sdei_list, list) {
 726                 if (event->type == SDEI_EVENT_TYPE_SHARED)
 727                         continue;
 728
 729                 if (event->reregister) {
 730                         CROSSCALL_INIT(arg, event);
 731                         /* call the cross-call function locally... */
 732                         _local_event_register(&arg);
 733                         if (arg.first_error)
 734                                 pr_err("Failed to re-register event %u: %d\n",
 735                                        event->event_num, arg.first_error);
 736                 }
 737
 738                 if (event->reenable) {
 739                         CROSSCALL_INIT(arg, event);
 740                         _local_event_enable(&arg);
 741                         if (arg.first_error)
 742                                 pr_err("Failed to re-enable event %u: %d\n",
 743                                        event->event_num, arg.first_error);
 744                 }
 745         }
 746         spin_unlock(&sdei_list_lock);
 747
 748         return sdei_unmask_local_cpu();
 749 }
 750
 751 /* When entering idle, mask/unmask events for this cpu */
 752 static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
 753                             void *data)
 754 {
 755         int rv;
 756
 757         switch (action) {
 758         case CPU_PM_ENTER:
 759                 rv = sdei_mask_local_cpu();
 760                 break;
 761         case CPU_PM_EXIT:
 762         case CPU_PM_ENTER_FAILED:
 763                 rv = sdei_unmask_local_cpu();
 764                 break;
 765         default:
 766                 return NOTIFY_DONE;
 767         }
 768
 769         if (rv)
 770                 return notifier_from_errno(rv);
 771
 772         return NOTIFY_OK;
 773 }
 774
 775 static struct notifier_block sdei_pm_nb = {
 776         .notifier_call = sdei_pm_notifier,
 777 };
 778
 779 static int sdei_device_suspend(struct device *dev)
 780 {
 781         on_each_cpu(_ipi_mask_cpu, NULL, true);
 782
 783         return 0;
 784 }
 785
 786 static int sdei_device_resume(struct device *dev)
 787 {
 788         on_each_cpu(_ipi_unmask_cpu, NULL, true);
 789
 790         return 0;
 791 }
 792
 793 /*
 794  * We need all events to be reregistered when we resume from hibernate.
 795  *
 796  * The sequence is freeze->thaw. Reboot. freeze->restore. We unregister
 797  * events during freeze, then re-register and re-enable them during thaw
 798  * and restore.
 799  */
 800 static int sdei_device_freeze(struct device *dev)
 801 {
 802         int err;
 803
 804         /* unregister private events */
 805         cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
 806
 807         err = sdei_unregister_shared();
 808         if (err)
 809                 return err;
 810
 811         return 0;
 812 }
 813
 814 static int sdei_device_thaw(struct device *dev)
 815 {
 816         int err;
 817
 818         /* re-register shared events */
 819         err = sdei_reregister_shared();
 820         if (err) {
 821                 pr_warn("Failed to re-register shared events...\n");
 822                 sdei_mark_interface_broken();
 823                 return err;
 824         }
 825
 826         err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
 827                                 &sdei_cpuhp_up, &sdei_cpuhp_down);
 828         if (err)
 829                 pr_warn("Failed to re-register CPU hotplug notifier...\n");
 830
 831         return err;
 832 }
 833
 834 static int sdei_device_restore(struct device *dev)
 835 {
 836         int err;
 837
 838         err = sdei_platform_reset();
 839         if (err)
 840                 return err;
 841
 842         return sdei_device_thaw(dev);
 843 }
 844
 845 static const struct dev_pm_ops sdei_pm_ops = {
 846         .suspend = sdei_device_suspend,
 847         .resume = sdei_device_resume,
 848         .freeze = sdei_device_freeze,
 849         .thaw = sdei_device_thaw,
 850         .restore = sdei_device_restore,
 851 };
 852
 853 /*
 854  * Mask all CPUs and unregister all events on panic, reboot or kexec.
 855  */
 856 static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
 857                                 void *data)
 858 {
 859         /*
 860          * We are going to reset the interface, after this there is no point
 861          * doing work when we take CPUs offline.
 862          */
 863         cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
 864
 865         sdei_platform_reset();
 866
 867         return NOTIFY_OK;
 868 }
 869
 870 static struct notifier_block sdei_reboot_nb = {
 871         .notifier_call = sdei_reboot_notifier,
 872 };
 873
 874 static void sdei_smccc_smc(unsigned long function_id,
 875                            unsigned long arg0, unsigned long arg1,
 876                            unsigned long arg2, unsigned long arg3,
 877                            unsigned long arg4, struct arm_smccc_res *res)
 878 {
 879         arm_smccc_smc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
 880 }
 881
 882 static void sdei_smccc_hvc(unsigned long function_id,
 883                            unsigned long arg0, unsigned long arg1,
 884                            unsigned long arg2, unsigned long arg3,
 885                            unsigned long arg4, struct arm_smccc_res *res)
 886 {
 887         arm_smccc_hvc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
 888 }
 889
 890 static int sdei_get_conduit(struct platform_device *pdev)
 891 {
 892         const char *method;
 893         struct device_node *np = pdev->dev.of_node;
 894
 895         sdei_firmware_call = NULL;
 896         if (np) {
 897                 if (of_property_read_string(np, "method", &method)) {
 898                         pr_warn("missing \"method\" property\n");
 899                         return CONDUIT_INVALID;
 900                 }
 901
 902                 if (!strcmp("hvc", method)) {
 903                         sdei_firmware_call = &sdei_smccc_hvc;
 904                         return CONDUIT_HVC;
 905                 } else if (!strcmp("smc", method)) {
 906                         sdei_firmware_call = &sdei_smccc_smc;
 907                         return CONDUIT_SMC;
 908                 }
 909
 910                 pr_warn("invalid \"method\" property: %s\n", method);
 911         } else if (IS_ENABLED(CONFIG_ACPI) && !acpi_disabled) {
 912                 if (acpi_psci_use_hvc()) {
 913                         sdei_firmware_call = &sdei_smccc_hvc;
 914                         return CONDUIT_HVC;
 915                 } else {
 916                         sdei_firmware_call = &sdei_smccc_smc;
 917                         return CONDUIT_SMC;
 918                 }
 919         }
 920
 921         return CONDUIT_INVALID;
 922 }
 923
 924 static int sdei_probe(struct platform_device *pdev)
 925 {
 926         int err;
 927         u64 ver = 0;
 928         int conduit;
 929
 930         conduit = sdei_get_conduit(pdev);
 931         if (!sdei_firmware_call)
 932                 return 0;
 933
 934         err = sdei_api_get_version(&ver);
 935         if (err == -EOPNOTSUPP)
 936                 pr_err("advertised but not implemented in platform firmware\n");
 937         if (err) {
 938                 pr_err("Failed to get SDEI version: %d\n", err);
 939                 sdei_mark_interface_broken();
 940                 return err;
 941         }
 942
 943         pr_info("SDEIv%d.%d (0x%x) detected in firmware.\n",
 944                 (int)SDEI_VERSION_MAJOR(ver), (int)SDEI_VERSION_MINOR(ver),
 945                 (int)SDEI_VERSION_VENDOR(ver));
 946
 947         if (SDEI_VERSION_MAJOR(ver) != 1) {
 948                 pr_warn("Conflicting SDEI version detected.\n");
 949                 sdei_mark_interface_broken();
 950                 return -EINVAL;
 951         }
 952
 953         err = sdei_platform_reset();
 954         if (err)
 955                 return err;
 956
 957         sdei_entry_point = sdei_arch_get_entry_point(conduit);
 958         if (!sdei_entry_point) {
 959                 /* Not supported due to hardware or boot configuration */
 960                 sdei_mark_interface_broken();
 961                 return 0;
 962         }
 963
 964         err = cpu_pm_register_notifier(&sdei_pm_nb);
 965         if (err) {
 966                 pr_warn("Failed to register CPU PM notifier...\n");
 967                 goto error;
 968         }
 969
 970         err = register_reboot_notifier(&sdei_reboot_nb);
 971         if (err) {
 972                 pr_warn("Failed to register reboot notifier...\n");
 973                 goto remove_cpupm;
 974         }
 975
 976         err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
 977                                 &sdei_cpuhp_up, &sdei_cpuhp_down);
 978         if (err) {
 979                 pr_warn("Failed to register CPU hotplug notifier...\n");
 980                 goto remove_reboot;
 981         }
 982
 983         return 0;
 984
 985 remove_reboot:
 986         unregister_reboot_notifier(&sdei_reboot_nb);
 987
 988 remove_cpupm:
 989         cpu_pm_unregister_notifier(&sdei_pm_nb);
 990
 991 error:
 992         sdei_mark_interface_broken();
 993         return err;
 994 }
 995
 996 static const struct of_device_id sdei_of_match[] = {
 997         { .compatible = "arm,sdei-1.0" },
 998         {}
 999 };
1000
1001 static struct platform_driver sdei_driver = {
1002         .driver         = {
1003                 .name                   = "sdei",
1004                 .pm                     = &sdei_pm_ops,
1005                 .of_match_table         = sdei_of_match,
1006         },
1007         .probe          = sdei_probe,
1008 };
1009
1010 static bool __init sdei_present_dt(void)
1011 {
1012         struct device_node *np, *fw_np;
1013
1014         fw_np = of_find_node_by_name(NULL, "firmware");
1015         if (!fw_np)
1016                 return false;
1017
1018         np = of_find_matching_node(fw_np, sdei_of_match);
1019         if (!np)
1020                 return false;
1021         of_node_put(np);
1022
1023         return true;
1024 }
1025
1026 static bool __init sdei_present_acpi(void)
1027 {
1028         acpi_status status;
1029         struct platform_device *pdev;
1030         struct acpi_table_header *sdei_table_header;
1031
1032         if (acpi_disabled)
1033                 return false;
1034
1035         status = acpi_get_table(ACPI_SIG_SDEI, 0, &sdei_table_header);
1036         if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
1037                 const char *msg = acpi_format_exception(status);
1038
1039                 pr_info("Failed to get ACPI:SDEI table, %s\n", msg);
1040         }
1041         if (ACPI_FAILURE(status))
1042                 return false;
1043
1044         pdev = platform_device_register_simple(sdei_driver.driver.name, 0, NULL,
1045                                                0);
1046         if (IS_ERR(pdev))
1047                 return false;
1048
1049         return true;
1050 }
1051
1052 static int __init sdei_init(void)
1053 {
1054         if (sdei_present_dt() || sdei_present_acpi())
1055                 platform_driver_register(&sdei_driver);
1056
1057         return 0;
1058 }
1059
1060 /*
1061  * On an ACPI system SDEI needs to be ready before HEST:GHES tries to register
1062  * its events. ACPI is initialised from a subsys_initcall(), GHES is initialised
1063  * by device_initcall(). We want to be called in the middle.
1064  */
1065 subsys_initcall_sync(sdei_init);
1066
1067 int sdei_event_handler(struct pt_regs *regs,
1068                        struct sdei_registered_event *arg)
1069 {
1070         int err;
1071         mm_segment_t orig_addr_limit;
1072         u32 event_num = arg->event_num;
1073
1074         orig_addr_limit = get_fs();
1075         set_fs(USER_DS);
1076
1077         err = arg->callback(event_num, regs, arg->callback_arg);
1078         if (err)
1079                 pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
1080                                    event_num, smp_processor_id(), err);
1081
1082         set_fs(orig_addr_limit);
1083
1084         return err;
1085 }
1086 NOKPROBE_SYMBOL(sdei_event_handler);