arch/x86/platform/efi/efi.c

   1 /*
   2  * Common EFI (Extensible Firmware Interface) support functions
   3  * Based on Extensible Firmware Interface Specification version 1.0
   4  *
   5  * Copyright (C) 1999 VA Linux Systems
   6  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
   7  * Copyright (C) 1999-2002 Hewlett-Packard Co.
   8  *      David Mosberger-Tang <davidm@hpl.hp.com>
   9  *      Stephane Eranian <eranian@hpl.hp.com>
  10  * Copyright (C) 2005-2008 Intel Co.
  11  *      Fenghua Yu <fenghua.yu@intel.com>
  12  *      Bibo Mao <bibo.mao@intel.com>
  13  *      Chandramouli Narayanan <mouli@linux.intel.com>
  14  *      Huang Ying <ying.huang@intel.com>
  15  * Copyright (C) 2013 SuSE Labs
  16  *      Borislav Petkov <bp@suse.de> - runtime services VA mapping
  17  *
  18  * Copied from efi_32.c to eliminate the duplicated code between EFI
  19  * 32/64 support code. --ying 2007-10-26
  20  *
  21  * All EFI Runtime Services are not implemented yet as EFI only
  22  * supports physical mode addressing on SoftSDV. This is to be fixed
  23  * in a future version.  --drummond 1999-07-20
  24  *
  25  * Implemented EFI runtime services and virtual mode calls.  --davidm
  26  *
  27  * Goutham Rao: <goutham.rao@intel.com>
  28  *      Skip non-WB memory and ignore empty memory ranges.
  29  */
  30
  31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  32
  33 #include <linux/kernel.h>
  34 #include <linux/init.h>
  35 #include <linux/efi.h>
  36 #include <linux/efi-bgrt.h>
  37 #include <linux/export.h>
  38 #include <linux/bootmem.h>
  39 #include <linux/slab.h>
  40 #include <linux/memblock.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/uaccess.h>
  43 #include <linux/time.h>
  44 #include <linux/io.h>
  45 #include <linux/reboot.h>
  46 #include <linux/bcd.h>
  47
  48 #include <asm/setup.h>
  49 #include <asm/efi.h>
  50 #include <asm/time.h>
  51 #include <asm/cacheflush.h>
  52 #include <asm/tlbflush.h>
  53 #include <asm/x86_init.h>
  54 #include <asm/uv/uv.h>
  55
  56 static struct efi efi_phys __initdata;
  57 static efi_system_table_t efi_systab __initdata;
  58
  59 static efi_config_table_type_t arch_tables[] __initdata = {
  60 #ifdef CONFIG_X86_UV
  61         {UV_SYSTEM_TABLE_GUID, "UVsystab", &efi.uv_systab},
  62 #endif
  63         {NULL_GUID, NULL, NULL},
  64 };
  65
  66 u64 efi_setup;          /* efi setup_data physical address */
  67
  68 static int add_efi_memmap __initdata;
  69 static int __init setup_add_efi_memmap(char *arg)
  70 {
  71         add_efi_memmap = 1;
  72         return 0;
  73 }
  74 early_param("add_efi_memmap", setup_add_efi_memmap);
  75
  76 static efi_status_t __init phys_efi_set_virtual_address_map(
  77         unsigned long memory_map_size,
  78         unsigned long descriptor_size,
  79         u32 descriptor_version,
  80         efi_memory_desc_t *virtual_map)
  81 {
  82         efi_status_t status;
  83         unsigned long flags;
  84         pgd_t *save_pgd;
  85
  86         save_pgd = efi_call_phys_prolog();
  87
  88         /* Disable interrupts around EFI calls: */
  89         local_irq_save(flags);
  90         status = efi_call_phys(efi_phys.set_virtual_address_map,
  91                                memory_map_size, descriptor_size,
  92                                descriptor_version, virtual_map);
  93         local_irq_restore(flags);
  94
  95         efi_call_phys_epilog(save_pgd);
  96
  97         return status;
  98 }
  99
 100 void __init efi_find_mirror(void)
 101 {
 102         efi_memory_desc_t *md;
 103         u64 mirror_size = 0, total_size = 0;
 104
 105         for_each_efi_memory_desc(md) {
 106                 unsigned long long start = md->phys_addr;
 107                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 108
 109                 total_size += size;
 110                 if (md->attribute & EFI_MEMORY_MORE_RELIABLE) {
 111                         memblock_mark_mirror(start, size);
 112                         mirror_size += size;
 113                 }
 114         }
 115         if (mirror_size)
 116                 pr_info("Memory: %lldM/%lldM mirrored memory\n",
 117                         mirror_size>>20, total_size>>20);
 118 }
 119
 120 /*
 121  * Tell the kernel about the EFI memory map.  This might include
 122  * more than the max 128 entries that can fit in the e820 legacy
 123  * (zeropage) memory map.
 124  */
 125
 126 static void __init do_add_efi_memmap(void)
 127 {
 128         efi_memory_desc_t *md;
 129
 130         for_each_efi_memory_desc(md) {
 131                 unsigned long long start = md->phys_addr;
 132                 unsigned long long size = md->num_pages << EFI_PAGE_SHIFT;
 133                 int e820_type;
 134
 135                 switch (md->type) {
 136                 case EFI_LOADER_CODE:
 137                 case EFI_LOADER_DATA:
 138                 case EFI_BOOT_SERVICES_CODE:
 139                 case EFI_BOOT_SERVICES_DATA:
 140                 case EFI_CONVENTIONAL_MEMORY:
 141                         if (md->attribute & EFI_MEMORY_WB)
 142                                 e820_type = E820_RAM;
 143                         else
 144                                 e820_type = E820_RESERVED;
 145                         break;
 146                 case EFI_ACPI_RECLAIM_MEMORY:
 147                         e820_type = E820_ACPI;
 148                         break;
 149                 case EFI_ACPI_MEMORY_NVS:
 150                         e820_type = E820_NVS;
 151                         break;
 152                 case EFI_UNUSABLE_MEMORY:
 153                         e820_type = E820_UNUSABLE;
 154                         break;
 155                 case EFI_PERSISTENT_MEMORY:
 156                         e820_type = E820_PMEM;
 157                         break;
 158                 default:
 159                         /*
 160                          * EFI_RESERVED_TYPE EFI_RUNTIME_SERVICES_CODE
 161                          * EFI_RUNTIME_SERVICES_DATA EFI_MEMORY_MAPPED_IO
 162                          * EFI_MEMORY_MAPPED_IO_PORT_SPACE EFI_PAL_CODE
 163                          */
 164                         e820_type = E820_RESERVED;
 165                         break;
 166                 }
 167                 e820_add_region(start, size, e820_type);
 168         }
 169         sanitize_e820_map(e820->map, ARRAY_SIZE(e820->map), &e820->nr_map);
 170 }
 171
 172 int __init efi_memblock_x86_reserve_range(void)
 173 {
 174         struct efi_info *e = &boot_params.efi_info;
 175         struct efi_memory_map_data data;
 176         phys_addr_t pmap;
 177         int rv;
 178
 179         if (efi_enabled(EFI_PARAVIRT))
 180                 return 0;
 181
 182 #ifdef CONFIG_X86_32
 183         /* Can't handle data above 4GB at this time */
 184         if (e->efi_memmap_hi) {
 185                 pr_err("Memory map is above 4GB, disabling EFI.\n");
 186                 return -EINVAL;
 187         }
 188         pmap =  e->efi_memmap;
 189 #else
 190         pmap = (e->efi_memmap | ((__u64)e->efi_memmap_hi << 32));
 191 #endif
 192         data.phys_map           = pmap;
 193         data.size               = e->efi_memmap_size;
 194         data.desc_size          = e->efi_memdesc_size;
 195         data.desc_version       = e->efi_memdesc_version;
 196
 197         rv = efi_memmap_init_early(&data);
 198         if (rv)
 199                 return rv;
 200
 201         if (add_efi_memmap)
 202                 do_add_efi_memmap();
 203
 204         WARN(efi.memmap.desc_version != 1,
 205              "Unexpected EFI_MEMORY_DESCRIPTOR version %ld",
 206              efi.memmap.desc_version);
 207
 208         memblock_reserve(pmap, efi.memmap.nr_map * efi.memmap.desc_size);
 209
 210         return 0;
 211 }
 212
 213 #define OVERFLOW_ADDR_SHIFT     (64 - EFI_PAGE_SHIFT)
 214 #define OVERFLOW_ADDR_MASK      (U64_MAX << OVERFLOW_ADDR_SHIFT)
 215 #define U64_HIGH_BIT            (~(U64_MAX >> 1))
 216
 217 static bool __init efi_memmap_entry_valid(const efi_memory_desc_t *md, int i)
 218 {
 219         u64 end = (md->num_pages << EFI_PAGE_SHIFT) + md->phys_addr - 1;
 220         u64 end_hi = 0;
 221         char buf[64];
 222
 223         if (md->num_pages == 0) {
 224                 end = 0;
 225         } else if (md->num_pages > EFI_PAGES_MAX ||
 226                    EFI_PAGES_MAX - md->num_pages <
 227                    (md->phys_addr >> EFI_PAGE_SHIFT)) {
 228                 end_hi = (md->num_pages & OVERFLOW_ADDR_MASK)
 229                         >> OVERFLOW_ADDR_SHIFT;
 230
 231                 if ((md->phys_addr & U64_HIGH_BIT) && !(end & U64_HIGH_BIT))
 232                         end_hi += 1;
 233         } else {
 234                 return true;
 235         }
 236
 237         pr_warn_once(FW_BUG "Invalid EFI memory map entries:\n");
 238
 239         if (end_hi) {
 240                 pr_warn("mem%02u: %s range=[0x%016llx-0x%llx%016llx] (invalid)\n",
 241                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 242                         md->phys_addr, end_hi, end);
 243         } else {
 244                 pr_warn("mem%02u: %s range=[0x%016llx-0x%016llx] (invalid)\n",
 245                         i, efi_md_typeattr_format(buf, sizeof(buf), md),
 246                         md->phys_addr, end);
 247         }
 248         return false;
 249 }
 250
 251 static void __init efi_clean_memmap(void)
 252 {
 253         efi_memory_desc_t *out = efi.memmap.map;
 254         const efi_memory_desc_t *in = out;
 255         const efi_memory_desc_t *end = efi.memmap.map_end;
 256         int i, n_removal;
 257
 258         for (i = n_removal = 0; in < end; i++) {
 259                 if (efi_memmap_entry_valid(in, i)) {
 260                         if (out != in)
 261                                 memcpy(out, in, efi.memmap.desc_size);
 262                         out = (void *)out + efi.memmap.desc_size;
 263                 } else {
 264                         n_removal++;
 265                 }
 266                 in = (void *)in + efi.memmap.desc_size;
 267         }
 268
 269         if (n_removal > 0) {
 270                 u64 size = efi.memmap.nr_map - n_removal;
 271
 272                 pr_warn("Removing %d invalid memory map entries.\n", n_removal);
 273                 efi_memmap_install(efi.memmap.phys_map, size);
 274         }
 275 }
 276
 277 void __init efi_print_memmap(void)
 278 {
 279         efi_memory_desc_t *md;
 280         int i = 0;
 281
 282         for_each_efi_memory_desc(md) {
 283                 char buf[64];
 284
 285                 pr_info("mem%02u: %s range=[0x%016llx-0x%016llx] (%lluMB)\n",
 286                         i++, efi_md_typeattr_format(buf, sizeof(buf), md),
 287                         md->phys_addr,
 288                         md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT) - 1,
 289                         (md->num_pages >> (20 - EFI_PAGE_SHIFT)));
 290         }
 291 }
 292
 293 static int __init efi_systab_init(void *phys)
 294 {
 295         if (efi_enabled(EFI_64BIT)) {
 296                 efi_system_table_64_t *systab64;
 297                 struct efi_setup_data *data = NULL;
 298                 u64 tmp = 0;
 299
 300                 if (efi_setup) {
 301                         data = early_memremap(efi_setup, sizeof(*data));
 302                         if (!data)
 303                                 return -ENOMEM;
 304                 }
 305                 systab64 = early_memremap((unsigned long)phys,
 306                                          sizeof(*systab64));
 307                 if (systab64 == NULL) {
 308                         pr_err("Couldn't map the system table!\n");
 309                         if (data)
 310                                 early_memunmap(data, sizeof(*data));
 311                         return -ENOMEM;
 312                 }
 313
 314                 efi_systab.hdr = systab64->hdr;
 315                 efi_systab.fw_vendor = data ? (unsigned long)data->fw_vendor :
 316                                               systab64->fw_vendor;
 317                 tmp |= data ? data->fw_vendor : systab64->fw_vendor;
 318                 efi_systab.fw_revision = systab64->fw_revision;
 319                 efi_systab.con_in_handle = systab64->con_in_handle;
 320                 tmp |= systab64->con_in_handle;
 321                 efi_systab.con_in = systab64->con_in;
 322                 tmp |= systab64->con_in;
 323                 efi_systab.con_out_handle = systab64->con_out_handle;
 324                 tmp |= systab64->con_out_handle;
 325                 efi_systab.con_out = systab64->con_out;
 326                 tmp |= systab64->con_out;
 327                 efi_systab.stderr_handle = systab64->stderr_handle;
 328                 tmp |= systab64->stderr_handle;
 329                 efi_systab.stderr = systab64->stderr;
 330                 tmp |= systab64->stderr;
 331                 efi_systab.runtime = data ?
 332                                      (void *)(unsigned long)data->runtime :
 333                                      (void *)(unsigned long)systab64->runtime;
 334                 tmp |= data ? data->runtime : systab64->runtime;
 335                 efi_systab.boottime = (void *)(unsigned long)systab64->boottime;
 336                 tmp |= systab64->boottime;
 337                 efi_systab.nr_tables = systab64->nr_tables;
 338                 efi_systab.tables = data ? (unsigned long)data->tables :
 339                                            systab64->tables;
 340                 tmp |= data ? data->tables : systab64->tables;
 341
 342                 early_memunmap(systab64, sizeof(*systab64));
 343                 if (data)
 344                         early_memunmap(data, sizeof(*data));
 345 #ifdef CONFIG_X86_32
 346                 if (tmp >> 32) {
 347                         pr_err("EFI data located above 4GB, disabling EFI.\n");
 348                         return -EINVAL;
 349                 }
 350 #endif
 351         } else {
 352                 efi_system_table_32_t *systab32;
 353
 354                 systab32 = early_memremap((unsigned long)phys,
 355                                          sizeof(*systab32));
 356                 if (systab32 == NULL) {
 357                         pr_err("Couldn't map the system table!\n");
 358                         return -ENOMEM;
 359                 }
 360
 361                 efi_systab.hdr = systab32->hdr;
 362                 efi_systab.fw_vendor = systab32->fw_vendor;
 363                 efi_systab.fw_revision = systab32->fw_revision;
 364                 efi_systab.con_in_handle = systab32->con_in_handle;
 365                 efi_systab.con_in = systab32->con_in;
 366                 efi_systab.con_out_handle = systab32->con_out_handle;
 367                 efi_systab.con_out = systab32->con_out;
 368                 efi_systab.stderr_handle = systab32->stderr_handle;
 369                 efi_systab.stderr = systab32->stderr;
 370                 efi_systab.runtime = (void *)(unsigned long)systab32->runtime;
 371                 efi_systab.boottime = (void *)(unsigned long)systab32->boottime;
 372                 efi_systab.nr_tables = systab32->nr_tables;
 373                 efi_systab.tables = systab32->tables;
 374
 375                 early_memunmap(systab32, sizeof(*systab32));
 376         }
 377
 378         efi.systab = &efi_systab;
 379
 380         /*
 381          * Verify the EFI Table
 382          */
 383         if (efi.systab->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) {
 384                 pr_err("System table signature incorrect!\n");
 385                 return -EINVAL;
 386         }
 387         if ((efi.systab->hdr.revision >> 16) == 0)
 388                 pr_err("Warning: System table version %d.%02d, expected 1.00 or greater!\n",
 389                        efi.systab->hdr.revision >> 16,
 390                        efi.systab->hdr.revision & 0xffff);
 391
 392         return 0;
 393 }
 394
 395 static int __init efi_runtime_init32(void)
 396 {
 397         efi_runtime_services_32_t *runtime;
 398
 399         runtime = early_memremap((unsigned long)efi.systab->runtime,
 400                         sizeof(efi_runtime_services_32_t));
 401         if (!runtime) {
 402                 pr_err("Could not map the runtime service table!\n");
 403                 return -ENOMEM;
 404         }
 405
 406         /*
 407          * We will only need *early* access to the SetVirtualAddressMap
 408          * EFI runtime service. All other runtime services will be called
 409          * via the virtual mapping.
 410          */
 411         efi_phys.set_virtual_address_map =
 412                         (efi_set_virtual_address_map_t *)
 413                         (unsigned long)runtime->set_virtual_address_map;
 414         early_memunmap(runtime, sizeof(efi_runtime_services_32_t));
 415
 416         return 0;
 417 }
 418
 419 static int __init efi_runtime_init64(void)
 420 {
 421         efi_runtime_services_64_t *runtime;
 422
 423         runtime = early_memremap((unsigned long)efi.systab->runtime,
 424                         sizeof(efi_runtime_services_64_t));
 425         if (!runtime) {
 426                 pr_err("Could not map the runtime service table!\n");
 427                 return -ENOMEM;
 428         }
 429
 430         /*
 431          * We will only need *early* access to the SetVirtualAddressMap
 432          * EFI runtime service. All other runtime services will be called
 433          * via the virtual mapping.
 434          */
 435         efi_phys.set_virtual_address_map =
 436                         (efi_set_virtual_address_map_t *)
 437                         (unsigned long)runtime->set_virtual_address_map;
 438         early_memunmap(runtime, sizeof(efi_runtime_services_64_t));
 439
 440         return 0;
 441 }
 442
 443 static int __init efi_runtime_init(void)
 444 {
 445         int rv;
 446
 447         /*
 448          * Check out the runtime services table. We need to map
 449          * the runtime services table so that we can grab the physical
 450          * address of several of the EFI runtime functions, needed to
 451          * set the firmware into virtual mode.
 452          *
 453          * When EFI_PARAVIRT is in force then we could not map runtime
 454          * service memory region because we do not have direct access to it.
 455          * However, runtime services are available through proxy functions
 456          * (e.g. in case of Xen dom0 EFI implementation they call special
 457          * hypercall which executes relevant EFI functions) and that is why
 458          * they are always enabled.
 459          */
 460
 461         if (!efi_enabled(EFI_PARAVIRT)) {
 462                 if (efi_enabled(EFI_64BIT))
 463                         rv = efi_runtime_init64();
 464                 else
 465                         rv = efi_runtime_init32();
 466
 467                 if (rv)
 468                         return rv;
 469         }
 470
 471         set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 472
 473         return 0;
 474 }
 475
 476 void __init efi_init(void)
 477 {
 478         efi_char16_t *c16;
 479         char vendor[100] = "unknown";
 480         int i = 0;
 481
 482 #ifdef CONFIG_X86_32
 483         if (boot_params.efi_info.efi_systab_hi ||
 484             boot_params.efi_info.efi_memmap_hi) {
 485                 pr_info("Table located above 4GB, disabling EFI.\n");
 486                 return;
 487         }
 488         efi_phys.systab = (efi_system_table_t *)boot_params.efi_info.efi_systab;
 489 #else
 490         efi_phys.systab = (efi_system_table_t *)
 491                           (boot_params.efi_info.efi_systab |
 492                           ((__u64)boot_params.efi_info.efi_systab_hi<<32));
 493 #endif
 494
 495         if (efi_systab_init(efi_phys.systab))
 496                 return;
 497
 498         efi.config_table = (unsigned long)efi.systab->tables;
 499         efi.fw_vendor    = (unsigned long)efi.systab->fw_vendor;
 500         efi.runtime      = (unsigned long)efi.systab->runtime;
 501
 502         /*
 503          * Show what we know for posterity
 504          */
 505         c16 = early_memremap_ro(efi.systab->fw_vendor,
 506                                 sizeof(vendor) * sizeof(efi_char16_t));
 507         if (c16) {
 508                 for (i = 0; i < sizeof(vendor) - 1 && c16[i]; ++i)
 509                         vendor[i] = c16[i];
 510                 vendor[i] = '\0';
 511                 early_memunmap(c16, sizeof(vendor) * sizeof(efi_char16_t));
 512         } else {
 513                 pr_err("Could not map the firmware vendor!\n");
 514         }
 515
 516         pr_info("EFI v%u.%.02u by %s\n",
 517                 efi.systab->hdr.revision >> 16,
 518                 efi.systab->hdr.revision & 0xffff, vendor);
 519
 520         if (efi_reuse_config(efi.systab->tables, efi.systab->nr_tables))
 521                 return;
 522
 523         if (efi_config_init(arch_tables))
 524                 return;
 525
 526         /*
 527          * Note: We currently don't support runtime services on an EFI
 528          * that doesn't match the kernel 32/64-bit mode.
 529          */
 530
 531         if (!efi_runtime_supported())
 532                 pr_info("No EFI runtime due to 32/64-bit mismatch with kernel\n");
 533         else {
 534                 if (efi_runtime_disabled() || efi_runtime_init()) {
 535                         efi_memmap_unmap();
 536                         return;
 537                 }
 538         }
 539
 540         efi_clean_memmap();
 541
 542         if (efi_enabled(EFI_DBG))
 543                 efi_print_memmap();
 544 }
 545
 546 void __init efi_late_init(void)
 547 {
 548         efi_bgrt_init();
 549 }
 550
 551 void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
 552 {
 553         u64 addr, npages;
 554
 555         addr = md->virt_addr;
 556         npages = md->num_pages;
 557
 558         memrange_efi_to_native(&addr, &npages);
 559
 560         if (executable)
 561                 set_memory_x(addr, npages);
 562         else
 563                 set_memory_nx(addr, npages);
 564 }
 565
 566 void __init runtime_code_page_mkexec(void)
 567 {
 568         efi_memory_desc_t *md;
 569
 570         /* Make EFI runtime service code area executable */
 571         for_each_efi_memory_desc(md) {
 572                 if (md->type != EFI_RUNTIME_SERVICES_CODE)
 573                         continue;
 574
 575                 efi_set_executable(md, true);
 576         }
 577 }
 578
 579 void __init efi_memory_uc(u64 addr, unsigned long size)
 580 {
 581         unsigned long page_shift = 1UL << EFI_PAGE_SHIFT;
 582         u64 npages;
 583
 584         npages = round_up(size, page_shift) / page_shift;
 585         memrange_efi_to_native(&addr, &npages);
 586         set_memory_uc(addr, npages);
 587 }
 588
 589 void __init old_map_region(efi_memory_desc_t *md)
 590 {
 591         u64 start_pfn, end_pfn, end;
 592         unsigned long size;
 593         void *va;
 594
 595         start_pfn = PFN_DOWN(md->phys_addr);
 596         size      = md->num_pages << PAGE_SHIFT;
 597         end       = md->phys_addr + size;
 598         end_pfn   = PFN_UP(end);
 599
 600         if (pfn_range_is_mapped(start_pfn, end_pfn)) {
 601                 va = __va(md->phys_addr);
 602
 603                 if (!(md->attribute & EFI_MEMORY_WB))
 604                         efi_memory_uc((u64)(unsigned long)va, size);
 605         } else
 606                 va = efi_ioremap(md->phys_addr, size,
 607                                  md->type, md->attribute);
 608
 609         md->virt_addr = (u64) (unsigned long) va;
 610         if (!va)
 611                 pr_err("ioremap of 0x%llX failed!\n",
 612                        (unsigned long long)md->phys_addr);
 613 }
 614
 615 /* Merge contiguous regions of the same type and attribute */
 616 static void __init efi_merge_regions(void)
 617 {
 618         efi_memory_desc_t *md, *prev_md = NULL;
 619
 620         for_each_efi_memory_desc(md) {
 621                 u64 prev_size;
 622
 623                 if (!prev_md) {
 624                         prev_md = md;
 625                         continue;
 626                 }
 627
 628                 if (prev_md->type != md->type ||
 629                     prev_md->attribute != md->attribute) {
 630                         prev_md = md;
 631                         continue;
 632                 }
 633
 634                 prev_size = prev_md->num_pages << EFI_PAGE_SHIFT;
 635
 636                 if (md->phys_addr == (prev_md->phys_addr + prev_size)) {
 637                         prev_md->num_pages += md->num_pages;
 638                         md->type = EFI_RESERVED_TYPE;
 639                         md->attribute = 0;
 640                         continue;
 641                 }
 642                 prev_md = md;
 643         }
 644 }
 645
 646 static void __init get_systab_virt_addr(efi_memory_desc_t *md)
 647 {
 648         unsigned long size;
 649         u64 end, systab;
 650
 651         size = md->num_pages << EFI_PAGE_SHIFT;
 652         end = md->phys_addr + size;
 653         systab = (u64)(unsigned long)efi_phys.systab;
 654         if (md->phys_addr <= systab && systab < end) {
 655                 systab += md->virt_addr - md->phys_addr;
 656                 efi.systab = (efi_system_table_t *)(unsigned long)systab;
 657         }
 658 }
 659
 660 static void *realloc_pages(void *old_memmap, int old_shift)
 661 {
 662         void *ret;
 663
 664         ret = (void *)__get_free_pages(GFP_KERNEL, old_shift + 1);
 665         if (!ret)
 666                 goto out;
 667
 668         /*
 669          * A first-time allocation doesn't have anything to copy.
 670          */
 671         if (!old_memmap)
 672                 return ret;
 673
 674         memcpy(ret, old_memmap, PAGE_SIZE << old_shift);
 675
 676 out:
 677         free_pages((unsigned long)old_memmap, old_shift);
 678         return ret;
 679 }
 680
 681 /*
 682  * Iterate the EFI memory map in reverse order because the regions
 683  * will be mapped top-down. The end result is the same as if we had
 684  * mapped things forward, but doesn't require us to change the
 685  * existing implementation of efi_map_region().
 686  */
 687 static inline void *efi_map_next_entry_reverse(void *entry)
 688 {
 689         /* Initial call */
 690         if (!entry)
 691                 return efi.memmap.map_end - efi.memmap.desc_size;
 692
 693         entry -= efi.memmap.desc_size;
 694         if (entry < efi.memmap.map)
 695                 return NULL;
 696
 697         return entry;
 698 }
 699
 700 /*
 701  * efi_map_next_entry - Return the next EFI memory map descriptor
 702  * @entry: Previous EFI memory map descriptor
 703  *
 704  * This is a helper function to iterate over the EFI memory map, which
 705  * we do in different orders depending on the current configuration.
 706  *
 707  * To begin traversing the memory map @entry must be %NULL.
 708  *
 709  * Returns %NULL when we reach the end of the memory map.
 710  */
 711 static void *efi_map_next_entry(void *entry)
 712 {
 713         if (!efi_enabled(EFI_OLD_MEMMAP) && efi_enabled(EFI_64BIT)) {
 714                 /*
 715                  * Starting in UEFI v2.5 the EFI_PROPERTIES_TABLE
 716                  * config table feature requires us to map all entries
 717                  * in the same order as they appear in the EFI memory
 718                  * map. That is to say, entry N must have a lower
 719                  * virtual address than entry N+1. This is because the
 720                  * firmware toolchain leaves relative references in
 721                  * the code/data sections, which are split and become
 722                  * separate EFI memory regions. Mapping things
 723                  * out-of-order leads to the firmware accessing
 724                  * unmapped addresses.
 725                  *
 726                  * Since we need to map things this way whether or not
 727                  * the kernel actually makes use of
 728                  * EFI_PROPERTIES_TABLE, let's just switch to this
 729                  * scheme by default for 64-bit.
 730                  */
 731                 return efi_map_next_entry_reverse(entry);
 732         }
 733
 734         /* Initial call */
 735         if (!entry)
 736                 return efi.memmap.map;
 737
 738         entry += efi.memmap.desc_size;
 739         if (entry >= efi.memmap.map_end)
 740                 return NULL;
 741
 742         return entry;
 743 }
 744
 745 static bool should_map_region(efi_memory_desc_t *md)
 746 {
 747         /*
 748          * Runtime regions always require runtime mappings (obviously).
 749          */
 750         if (md->attribute & EFI_MEMORY_RUNTIME)
 751                 return true;
 752
 753         /*
 754          * 32-bit EFI doesn't suffer from the bug that requires us to
 755          * reserve boot services regions, and mixed mode support
 756          * doesn't exist for 32-bit kernels.
 757          */
 758         if (IS_ENABLED(CONFIG_X86_32))
 759                 return false;
 760
 761         /*
 762          * Map all of RAM so that we can access arguments in the 1:1
 763          * mapping when making EFI runtime calls.
 764          */
 765         if (IS_ENABLED(CONFIG_EFI_MIXED) && !efi_is_native()) {
 766                 if (md->type == EFI_CONVENTIONAL_MEMORY ||
 767                     md->type == EFI_LOADER_DATA ||
 768                     md->type == EFI_LOADER_CODE)
 769                         return true;
 770         }
 771
 772         /*
 773          * Map boot services regions as a workaround for buggy
 774          * firmware that accesses them even when they shouldn't.
 775          *
 776          * See efi_{reserve,free}_boot_services().
 777          */
 778         if (md->type == EFI_BOOT_SERVICES_CODE ||
 779             md->type == EFI_BOOT_SERVICES_DATA)
 780                 return true;
 781
 782         return false;
 783 }
 784
 785 /*
 786  * Map the efi memory ranges of the runtime services and update new_mmap with
 787  * virtual addresses.
 788  */
 789 static void * __init efi_map_regions(int *count, int *pg_shift)
 790 {
 791         void *p, *new_memmap = NULL;
 792         unsigned long left = 0;
 793         unsigned long desc_size;
 794         efi_memory_desc_t *md;
 795
 796         desc_size = efi.memmap.desc_size;
 797
 798         p = NULL;
 799         while ((p = efi_map_next_entry(p))) {
 800                 md = p;
 801
 802                 if (!should_map_region(md))
 803                         continue;
 804
 805                 efi_map_region(md);
 806                 get_systab_virt_addr(md);
 807
 808                 if (left < desc_size) {
 809                         new_memmap = realloc_pages(new_memmap, *pg_shift);
 810                         if (!new_memmap)
 811                                 return NULL;
 812
 813                         left += PAGE_SIZE << *pg_shift;
 814                         (*pg_shift)++;
 815                 }
 816
 817                 memcpy(new_memmap + (*count * desc_size), md, desc_size);
 818
 819                 left -= desc_size;
 820                 (*count)++;
 821         }
 822
 823         return new_memmap;
 824 }
 825
 826 static void __init kexec_enter_virtual_mode(void)
 827 {
 828 #ifdef CONFIG_KEXEC_CORE
 829         efi_memory_desc_t *md;
 830         unsigned int num_pages;
 831
 832         efi.systab = NULL;
 833
 834         /*
 835          * We don't do virtual mode, since we don't do runtime services, on
 836          * non-native EFI. With efi=old_map, we don't do runtime services in
 837          * kexec kernel because in the initial boot something else might
 838          * have been mapped at these virtual addresses.
 839          */
 840         if (!efi_is_native() || efi_enabled(EFI_OLD_MEMMAP)) {
 841                 efi_memmap_unmap();
 842                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 843                 return;
 844         }
 845
 846         if (efi_alloc_page_tables()) {
 847                 pr_err("Failed to allocate EFI page tables\n");
 848                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 849                 return;
 850         }
 851
 852         /*
 853         * Map efi regions which were passed via setup_data. The virt_addr is a
 854         * fixed addr which was used in first kernel of a kexec boot.
 855         */
 856         for_each_efi_memory_desc(md) {
 857                 efi_map_region_fixed(md); /* FIXME: add error handling */
 858                 get_systab_virt_addr(md);
 859         }
 860
 861         /*
 862          * Unregister the early EFI memmap from efi_init() and install
 863          * the new EFI memory map.
 864          */
 865         efi_memmap_unmap();
 866
 867         if (efi_memmap_init_late(efi.memmap.phys_map,
 868                                  efi.memmap.desc_size * efi.memmap.nr_map)) {
 869                 pr_err("Failed to remap late EFI memory map\n");
 870                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 871                 return;
 872         }
 873
 874         BUG_ON(!efi.systab);
 875
 876         num_pages = ALIGN(efi.memmap.nr_map * efi.memmap.desc_size, PAGE_SIZE);
 877         num_pages >>= PAGE_SHIFT;
 878
 879         if (efi_setup_page_tables(efi.memmap.phys_map, num_pages)) {
 880                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 881                 return;
 882         }
 883
 884         efi_sync_low_kernel_mappings();
 885
 886         /*
 887          * Now that EFI is in virtual mode, update the function
 888          * pointers in the runtime service table to the new virtual addresses.
 889          *
 890          * Call EFI services through wrapper functions.
 891          */
 892         efi.runtime_version = efi_systab.hdr.revision;
 893
 894         efi_native_runtime_setup();
 895
 896         efi.set_virtual_address_map = NULL;
 897
 898         if (efi_enabled(EFI_OLD_MEMMAP) && (__supported_pte_mask & _PAGE_NX))
 899                 runtime_code_page_mkexec();
 900 #endif
 901 }
 902
 903 /*
 904  * This function will switch the EFI runtime services to virtual mode.
 905  * Essentially, we look through the EFI memmap and map every region that
 906  * has the runtime attribute bit set in its memory descriptor into the
 907  * efi_pgd page table.
 908  *
 909  * The old method which used to update that memory descriptor with the
 910  * virtual address obtained from ioremap() is still supported when the
 911  * kernel is booted with efi=old_map on its command line. Same old
 912  * method enabled the runtime services to be called without having to
 913  * thunk back into physical mode for every invocation.
 914  *
 915  * The new method does a pagetable switch in a preemption-safe manner
 916  * so that we're in a different address space when calling a runtime
 917  * function. For function arguments passing we do copy the PUDs of the
 918  * kernel page table into efi_pgd prior to each call.
 919  *
 920  * Specially for kexec boot, efi runtime maps in previous kernel should
 921  * be passed in via setup_data. In that case runtime ranges will be mapped
 922  * to the same virtual addresses as the first kernel, see
 923  * kexec_enter_virtual_mode().
 924  */
 925 static void __init __efi_enter_virtual_mode(void)
 926 {
 927         int count = 0, pg_shift = 0;
 928         void *new_memmap = NULL;
 929         efi_status_t status;
 930         unsigned long pa;
 931
 932         efi.systab = NULL;
 933
 934         if (efi_alloc_page_tables()) {
 935                 pr_err("Failed to allocate EFI page tables\n");
 936                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 937                 return;
 938         }
 939
 940         efi_merge_regions();
 941         new_memmap = efi_map_regions(&count, &pg_shift);
 942         if (!new_memmap) {
 943                 pr_err("Error reallocating memory, EFI runtime non-functional!\n");
 944                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 945                 return;
 946         }
 947
 948         pa = __pa(new_memmap);
 949
 950         /*
 951          * Unregister the early EFI memmap from efi_init() and install
 952          * the new EFI memory map that we are about to pass to the
 953          * firmware via SetVirtualAddressMap().
 954          */
 955         efi_memmap_unmap();
 956
 957         if (efi_memmap_init_late(pa, efi.memmap.desc_size * count)) {
 958                 pr_err("Failed to remap late EFI memory map\n");
 959                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 960                 return;
 961         }
 962
 963         BUG_ON(!efi.systab);
 964
 965         if (efi_setup_page_tables(pa, 1 << pg_shift)) {
 966                 clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 967                 return;
 968         }
 969
 970         efi_sync_low_kernel_mappings();
 971
 972         if (efi_is_native()) {
 973                 status = phys_efi_set_virtual_address_map(
 974                                 efi.memmap.desc_size * count,
 975                                 efi.memmap.desc_size,
 976                                 efi.memmap.desc_version,
 977                                 (efi_memory_desc_t *)pa);
 978         } else {
 979                 status = efi_thunk_set_virtual_address_map(
 980                                 efi_phys.set_virtual_address_map,
 981                                 efi.memmap.desc_size * count,
 982                                 efi.memmap.desc_size,
 983                                 efi.memmap.desc_version,
 984                                 (efi_memory_desc_t *)pa);
 985         }
 986
 987         if (status != EFI_SUCCESS) {
 988                 pr_alert("Unable to switch EFI into virtual mode (status=%lx)!\n",
 989                          status);
 990                 panic("EFI call to SetVirtualAddressMap() failed!");
 991         }
 992
 993         /*
 994          * Now that EFI is in virtual mode, update the function
 995          * pointers in the runtime service table to the new virtual addresses.
 996          *
 997          * Call EFI services through wrapper functions.
 998          */
 999         efi.runtime_version = efi_systab.hdr.revision;
1000
1001         if (efi_is_native())
1002                 efi_native_runtime_setup();
1003         else
1004                 efi_thunk_runtime_setup();
1005
1006         efi.set_virtual_address_map = NULL;
1007
1008         /*
1009          * Apply more restrictive page table mapping attributes now that
1010          * SVAM() has been called and the firmware has performed all
1011          * necessary relocation fixups for the new virtual addresses.
1012          */
1013         efi_runtime_update_mappings();
1014         efi_dump_pagetable();
1015
1016         /* clean DUMMY object */
1017         efi_delete_dummy_variable();
1018 }
1019
1020 void __init efi_enter_virtual_mode(void)
1021 {
1022         if (efi_enabled(EFI_PARAVIRT))
1023                 return;
1024
1025         if (efi_setup)
1026                 kexec_enter_virtual_mode();
1027         else
1028                 __efi_enter_virtual_mode();
1029 }
1030
1031 /*
1032  * Convenience functions to obtain memory types and attributes
1033  */
1034 u32 efi_mem_type(unsigned long phys_addr)
1035 {
1036         efi_memory_desc_t *md;
1037
1038         if (!efi_enabled(EFI_MEMMAP))
1039                 return 0;
1040
1041         for_each_efi_memory_desc(md) {
1042                 if ((md->phys_addr <= phys_addr) &&
1043                     (phys_addr < (md->phys_addr +
1044                                   (md->num_pages << EFI_PAGE_SHIFT))))
1045                         return md->type;
1046         }
1047         return 0;
1048 }
1049
1050 static int __init arch_parse_efi_cmdline(char *str)
1051 {
1052         if (!str) {
1053                 pr_warn("need at least one option\n");
1054                 return -EINVAL;
1055         }
1056
1057         if (parse_option_str(str, "old_map"))
1058                 set_bit(EFI_OLD_MEMMAP, &efi.flags);
1059
1060         return 0;
1061 }
1062 early_param("efi", arch_parse_efi_cmdline);