From 96045ed486b09b2dc21c3d09edb13f22dff875bf Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Mon, 27 Apr 2015 22:53:08 +0200 Subject: arm64: Mark PMU interrupt IRQF_NO_THREAD Mark the PMU interrupts as non-threadable, as is the case with arch/arm: d9c3365 ARM: 7813/1: Mark pmu interupt IRQF_NO_THREAD Acked-by: Will Deacon Suggested-by: Sebastian Andrzej Siewior Signed-off-by: Anders Roxell Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index cce18c8..702591f 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -488,7 +488,7 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu) } err = request_irq(irq, armpmu->handle_irq, - IRQF_NOBALANCING, + IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu", armpmu); if (err) { pr_err("unable to request IRQ%d for ARM PMU counters\n", -- cgit v0.10.2 From e8557d1f0c4d06260b8aed5d2400806a8e7ac21c Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Mon, 27 Apr 2015 22:53:09 +0200 Subject: arm64: Allow forced irq threading Now its safe to allow forced interrupt threading for arm64, all timer interrupts and the perf interrupt are marked NO_THREAD, as is the case with arch/arm: da0ec6f ARM: 7814/2: Allow forced irq threading Acked-by: Sebastian Andrzej Siewior Suggested-by: Sebastian Andrzej Siewior Signed-off-by: Anders Roxell Signed-off-by: Catalin Marinas diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 7796af4..99930cf 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -71,6 +71,7 @@ config ARM64 select HAVE_RCU_TABLE_FREE select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN + select IRQ_FORCED_THREADING select MODULES_USE_ELF_RELA select NO_BOOTMEM select OF -- cgit v0.10.2 From 68234df4ea7939f98431aa81113fbdce10c4a84b Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 20 Apr 2015 10:24:35 +0100 Subject: arm64: kill flush_cache_all() The documented semantics of flush_cache_all are not possible to provide for arm64 (short of flushing the entire physical address space by VA), and there are currently no users; KVM uses VA maintenance exclusively, cpu_reset is never called, and the only two users outside of arch code cannot be built for arm64. While cpu_soft_reset and related functions (which call flush_cache_all) were thought to be useful for kexec, their current implementations only serve to mask bugs. For correctness kexec will need to perform maintenance by VA anyway to account for system caches, line migration, and other subtleties of the cache architecture. As the extent of this cache maintenance will be kexec-specific, it should probably live in the kexec code. This patch removes flush_cache_all, and related unused components, preventing further abuse. Signed-off-by: Mark Rutland Cc: AKASHI Takahiro Cc: Geoff Levand Acked-by: Ard Biesheuvel Acked-by: Catalin Marinas Acked-by: Lorenzo Pieralisi Acked-by: Marc Zyngier Acked-by: Will Deacon Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index 67d309c..c75b8d0 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -40,10 +40,6 @@ * the implementation assumes non-aliasing VIPT D-cache and (aliasing) * VIPT or ASID-tagged VIVT I-cache. * - * flush_cache_all() - * - * Unconditionally clean and invalidate the entire cache. - * * flush_cache_mm(mm) * * Clean and invalidate all user space cache entries @@ -69,7 +65,6 @@ * - kaddr - page address * - size - region size */ -extern void flush_cache_all(void); extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void flush_icache_range(unsigned long start, unsigned long end); extern void __flush_dcache_area(void *addr, size_t len); diff --git a/arch/arm64/include/asm/proc-fns.h b/arch/arm64/include/asm/proc-fns.h index 220633b7..14ad6e4 100644 --- a/arch/arm64/include/asm/proc-fns.h +++ b/arch/arm64/include/asm/proc-fns.h @@ -28,12 +28,8 @@ struct mm_struct; struct cpu_suspend_ctx; -extern void cpu_cache_off(void); extern void cpu_do_idle(void); extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); -extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); -void cpu_soft_restart(phys_addr_t cpu_reset, - unsigned long addr) __attribute__((noreturn)); extern void cpu_do_suspend(struct cpu_suspend_ctx *ptr); extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr); diff --git a/arch/arm64/include/asm/system_misc.h b/arch/arm64/include/asm/system_misc.h index 7a18fab..659fbf5 100644 --- a/arch/arm64/include/asm/system_misc.h +++ b/arch/arm64/include/asm/system_misc.h @@ -41,7 +41,6 @@ struct mm_struct; extern void show_pte(struct mm_struct *mm, unsigned long addr); extern void __show_regs(struct pt_regs *); -void soft_restart(unsigned long); extern void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); #define UDBG_UNDEFINED (1 << 0) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index c6b1f3b..c506bee 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -58,14 +58,6 @@ unsigned long __stack_chk_guard __read_mostly; EXPORT_SYMBOL(__stack_chk_guard); #endif -void soft_restart(unsigned long addr) -{ - setup_mm_for_reboot(); - cpu_soft_restart(virt_to_phys(cpu_reset), addr); - /* Should never get here */ - BUG(); -} - /* * Function pointers to optional machine specific functions */ @@ -136,9 +128,7 @@ void machine_power_off(void) /* * Restart requires that the secondary CPUs stop performing any activity - * while the primary CPU resets the system. Systems with a single CPU can - * use soft_restart() as their machine descriptor's .restart hook, since that - * will cause the only available CPU to reset. Systems with multiple CPUs must + * while the primary CPU resets the system. Systems with multiple CPUs must * provide a HW restart implementation, to ensure that all CPUs reset at once. * This is required so that any code running after reset on the primary CPU * doesn't have to co-ordinate with other CPUs to ensure they aren't still diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 2560e1e..f563e9a 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -27,79 +27,6 @@ #include "proc-macros.S" /* - * __flush_dcache_all() - * - * Flush the whole D-cache. - * - * Corrupted registers: x0-x7, x9-x11 - */ -__flush_dcache_all: - dmb sy // ensure ordering with previous memory accesses - mrs x0, clidr_el1 // read clidr - and x3, x0, #0x7000000 // extract loc from clidr - lsr x3, x3, #23 // left align loc bit field - cbz x3, finished // if loc is 0, then no need to clean - mov x10, #0 // start clean at cache level 0 -loop1: - add x2, x10, x10, lsr #1 // work out 3x current cache level - lsr x1, x0, x2 // extract cache type bits from clidr - and x1, x1, #7 // mask of the bits for current cache only - cmp x1, #2 // see what cache we have at this level - b.lt skip // skip if no cache, or just i-cache - save_and_disable_irqs x9 // make CSSELR and CCSIDR access atomic - msr csselr_el1, x10 // select current cache level in csselr - isb // isb to sych the new cssr&csidr - mrs x1, ccsidr_el1 // read the new ccsidr - restore_irqs x9 - and x2, x1, #7 // extract the length of the cache lines - add x2, x2, #4 // add 4 (line length offset) - mov x4, #0x3ff - and x4, x4, x1, lsr #3 // find maximum number on the way size - clz w5, w4 // find bit position of way size increment - mov x7, #0x7fff - and x7, x7, x1, lsr #13 // extract max number of the index size -loop2: - mov x9, x4 // create working copy of max way size -loop3: - lsl x6, x9, x5 - orr x11, x10, x6 // factor way and cache number into x11 - lsl x6, x7, x2 - orr x11, x11, x6 // factor index number into x11 - dc cisw, x11 // clean & invalidate by set/way - subs x9, x9, #1 // decrement the way - b.ge loop3 - subs x7, x7, #1 // decrement the index - b.ge loop2 -skip: - add x10, x10, #2 // increment cache number - cmp x3, x10 - b.gt loop1 -finished: - mov x10, #0 // swith back to cache level 0 - msr csselr_el1, x10 // select current cache level in csselr - dsb sy - isb - ret -ENDPROC(__flush_dcache_all) - -/* - * flush_cache_all() - * - * Flush the entire cache system. The data cache flush is now achieved - * using atomic clean / invalidates working outwards from L1 cache. This - * is done using Set/Way based cache maintainance instructions. The - * instruction cache can still be invalidated back to the point of - * unification in a single instruction. - */ -ENTRY(flush_cache_all) - mov x12, lr - bl __flush_dcache_all - mov x0, #0 - ic ialluis // I+BTB cache invalidate - ret x12 -ENDPROC(flush_cache_all) - -/* * flush_icache_range(start,end) * * Ensure that the I and D caches are coherent within specified region. diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index b6f14e8..4dfa397 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -102,7 +102,6 @@ EXPORT_SYMBOL(flush_dcache_page); /* * Additional functions defined in assembly. */ -EXPORT_SYMBOL(flush_cache_all); EXPORT_SYMBOL(flush_icache_range); #ifdef CONFIG_TRANSPARENT_HUGEPAGE diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index cdd754e..39139a3 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -46,52 +46,6 @@ #define MAIR(attr, mt) ((attr) << ((mt) * 8)) /* - * cpu_cache_off() - * - * Turn the CPU D-cache off. - */ -ENTRY(cpu_cache_off) - mrs x0, sctlr_el1 - bic x0, x0, #1 << 2 // clear SCTLR.C - msr sctlr_el1, x0 - isb - ret -ENDPROC(cpu_cache_off) - -/* - * cpu_reset(loc) - * - * Perform a soft reset of the system. Put the CPU into the same state - * as it would be if it had been reset, and branch to what would be the - * reset vector. It must be executed with the flat identity mapping. - * - * - loc - location to jump to for soft reset - */ - .align 5 -ENTRY(cpu_reset) - mrs x1, sctlr_el1 - bic x1, x1, #1 - msr sctlr_el1, x1 // disable the MMU - isb - ret x0 -ENDPROC(cpu_reset) - -ENTRY(cpu_soft_restart) - /* Save address of cpu_reset() and reset address */ - mov x19, x0 - mov x20, x1 - - /* Turn D-cache off */ - bl cpu_cache_off - - /* Push out all dirty data, and ensure cache is empty */ - bl flush_cache_all - - mov x0, x20 - ret x19 -ENDPROC(cpu_soft_restart) - -/* * cpu_do_idle() * * Idle the processor (wait for interrupt). -- cgit v0.10.2 From e985ad17cbbb55e10e2139eb25c087ef92fcdd48 Mon Sep 17 00:00:00 2001 From: Michal Simek Date: Mon, 18 May 2015 13:10:48 +0200 Subject: arm64: Rename temp variable in read*_relaxed() This resolves the following sparse warning from readl() and other macros, which ends up embedding readl_relaxed() using the same variable. Warning log: include/asm-generic/io.h:364:16: warning: symbol '__v' shadows an earlier one include/asm-generic/io.h:364:16: originally declared here include/asm-generic/io.h:372:16: warning: symbol '__v' shadows an earlier one include/asm-generic/io.h:372:16: originally declared here include/asm-generic/io.h:380:16: warning: symbol '__v' shadows an earlier one include/asm-generic/io.h:380:16: originally declared here include/asm-generic/io.h:568:16: warning: symbol '__v' shadows an earlier one include/asm-generic/io.h:568:16: originally declared here include/asm-generic/io.h:576:16: warning: symbol '__v' shadows an earlier one include/asm-generic/io.h:576:16: originally declared here include/asm-generic/io.h:584:16: warning: symbol '__v' shadows an earlier one include/asm-generic/io.h:584:16: originally declared here The same patch was already applied to arm32 as "ARM: 7118/1: rename temp variable in read*_relaxed()" (sha1: b0c1264f534a1cb3c52036a23a04d238434a0df6) Acked-by: Liviu Dudau Signed-off-by: Michal Simek Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 540f7c0..cb18715 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -117,10 +117,10 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) * ordering rules but do not guarantee any ordering relative to Normal memory * accesses. */ -#define readb_relaxed(c) ({ u8 __v = __raw_readb(c); __v; }) -#define readw_relaxed(c) ({ u16 __v = le16_to_cpu((__force __le16)__raw_readw(c)); __v; }) -#define readl_relaxed(c) ({ u32 __v = le32_to_cpu((__force __le32)__raw_readl(c)); __v; }) -#define readq_relaxed(c) ({ u64 __v = le64_to_cpu((__force __le64)__raw_readq(c)); __v; }) +#define readb_relaxed(c) ({ u8 __r = __raw_readb(c); __r; }) +#define readw_relaxed(c) ({ u16 __r = le16_to_cpu((__force __le16)__raw_readw(c)); __r; }) +#define readl_relaxed(c) ({ u32 __r = le32_to_cpu((__force __le32)__raw_readl(c)); __r; }) +#define readq_relaxed(c) ({ u64 __r = le64_to_cpu((__force __le64)__raw_readq(c)); __r; }) #define writeb_relaxed(v,c) ((void)__raw_writeb((v),(c))) #define writew_relaxed(v,c) ((void)__raw_writew((__force u16)cpu_to_le16(v),(c))) -- cgit v0.10.2 From 819a88263d5dbe398edd59cc1cf725ed1fdcfd79 Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Wed, 13 May 2015 14:12:46 +0100 Subject: ARM64: kernel: make cpu_ops hooks DT agnostic ARM64 CPU operations such as cpu_init and cpu_init_idle take a struct device_node pointer as a parameter, which corresponds to the device tree node of the logical cpu on which the operation has to be applied. With the advent of ACPI on arm64, where MADT static table entries are used to initialize cpus, the device tree node parameter in cpu_ops hooks become useless when booting with ACPI, since in that case cpu device tree nodes are not present and can not be used for cpu initialization. The current cpu_init hook requires a struct device_node pointer parameter because it is called while parsing the device tree to initialize CPUs, when the cpu_logical_map (that is used to match a cpu node reg property to a device tree node) for a given logical cpu id is not set up yet. This means that the cpu_init hook cannot rely on the of_get_cpu_node function to retrieve the device tree node corresponding to the logical cpu id passed in as parameter, so the cpu device tree node must be passed in as a parameter to fix this catch-22 dependency cycle. This patch reshuffles the cpu_logical_map initialization code so that the cpu_init cpu_ops hook can safely use the of_get_cpu_node function to retrieve the cpu device tree node, removing the need for the device tree node pointer parameter. In the process, the patch removes device tree node parameters from all cpu_ops hooks, in preparation for SMP DT/ACPI cpus initialization consolidation. Signed-off-by: Lorenzo Pieralisi Acked-by: Hanjun Guo Acked-by: Sudeep Holla Acked-by: Mark Rutland Tested-by: Hanjun Guo Tested-by: Mark Rutland [DT] Cc: Will Deacon Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index 5a31d67..7d8b999 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -19,15 +19,15 @@ #include #include -struct device_node; - /** * struct cpu_operations - Callback operations for hotplugging CPUs. * * @name: Name of the property as appears in a devicetree cpu node's - * enable-method property. - * @cpu_init: Reads any data necessary for a specific enable-method from the - * devicetree, for a given cpu node and proposed logical id. + * enable-method property. On systems booting with ACPI, @name + * identifies the struct cpu_operations entry corresponding to + * the boot protocol specified in the ACPI MADT table. + * @cpu_init: Reads any data necessary for a specific enable-method for a + * proposed logical id. * @cpu_prepare: Early one-time preparation step for a cpu. If there is a * mechanism for doing so, tests whether it is possible to boot * the given CPU. @@ -40,15 +40,15 @@ struct device_node; * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the * cpu being killed. * @cpu_kill: Ensures a cpu has left the kernel. Called from another cpu. - * @cpu_init_idle: Reads any data necessary to initialize CPU idle states from - * devicetree, for a given cpu node and proposed logical id. + * @cpu_init_idle: Reads any data necessary to initialize CPU idle states for + * a proposed logical id. * @cpu_suspend: Suspends a cpu and saves the required context. May fail owing * to wrong parameters or error conditions. Called from the * CPU being suspended. Must be called with IRQs disabled. */ struct cpu_operations { const char *name; - int (*cpu_init)(struct device_node *, unsigned int); + int (*cpu_init)(unsigned int); int (*cpu_prepare)(unsigned int); int (*cpu_boot)(unsigned int); void (*cpu_postboot)(void); @@ -58,14 +58,18 @@ struct cpu_operations { int (*cpu_kill)(unsigned int cpu); #endif #ifdef CONFIG_CPU_IDLE - int (*cpu_init_idle)(struct device_node *, unsigned int); + int (*cpu_init_idle)(unsigned int); int (*cpu_suspend)(unsigned long); #endif }; extern const struct cpu_operations *cpu_ops[NR_CPUS]; -int __init cpu_read_ops(struct device_node *dn, int cpu); -void __init cpu_read_bootcpu_ops(void); +int __init cpu_read_ops(int cpu); const struct cpu_operations *cpu_get_ops(const char *name); +static inline void __init cpu_read_bootcpu_ops(void) +{ + cpu_read_ops(0); +} + #endif /* ifndef __ASM_CPU_OPS_H */ diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index 8b83955..945cc1f 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -154,7 +154,7 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) if (!cpu_ops[enabled_cpus]) return; - if (cpu_ops[enabled_cpus]->cpu_init(NULL, enabled_cpus)) + if (cpu_ops[enabled_cpus]->cpu_init(enabled_cpus)) return; /* map the logical cpu id to cpu MPIDR */ diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index fb8ff9b..bbda2d2 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -52,9 +52,18 @@ const struct cpu_operations * __init cpu_get_ops(const char *name) /* * Read a cpu's enable method from the device tree and record it in cpu_ops. */ -int __init cpu_read_ops(struct device_node *dn, int cpu) +int __init cpu_read_ops(int cpu) { - const char *enable_method = of_get_property(dn, "enable-method", NULL); + const char *enable_method; + struct device_node *dn = of_get_cpu_node(cpu, NULL); + + if (!dn) { + if (!cpu) + pr_err("Failed to find device node for boot cpu\n"); + return -ENODEV; + } + + enable_method = of_get_property(dn, "enable-method", NULL); if (!enable_method) { /* * The boot CPU may not have an enable method (e.g. when @@ -75,13 +84,3 @@ int __init cpu_read_ops(struct device_node *dn, int cpu) return 0; } - -void __init cpu_read_bootcpu_ops(void) -{ - struct device_node *dn = of_get_cpu_node(0, NULL); - if (!dn) { - pr_err("Failed to find device node for boot cpu\n"); - return; - } - cpu_read_ops(dn, 0); -} diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c index a78143a..f8aa169 100644 --- a/arch/arm64/kernel/cpuidle.c +++ b/arch/arm64/kernel/cpuidle.c @@ -18,15 +18,10 @@ int arm_cpuidle_init(unsigned int cpu) { int ret = -EOPNOTSUPP; - struct device_node *cpu_node = of_cpu_device_node_get(cpu); - - if (!cpu_node) - return -ENODEV; if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_init_idle) - ret = cpu_ops[cpu]->cpu_init_idle(cpu_node, cpu); + ret = cpu_ops[cpu]->cpu_init_idle(cpu); - of_node_put(cpu_node); return ret; } diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index ea18cb5..efe3480 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -186,12 +186,15 @@ static int psci_migrate_info_type(void) return err; } -static int __maybe_unused cpu_psci_cpu_init_idle(struct device_node *cpu_node, - unsigned int cpu) +static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) { int i, ret, count = 0; struct psci_power_state *psci_states; - struct device_node *state_node; + struct device_node *state_node, *cpu_node; + + cpu_node = of_get_cpu_node(cpu, NULL); + if (!cpu_node) + return -ENODEV; /* * If the PSCI cpu_suspend function hook has not been initialized @@ -444,7 +447,7 @@ int __init psci_acpi_init(void) #ifdef CONFIG_SMP -static int __init cpu_psci_cpu_init(struct device_node *dn, unsigned int cpu) +static int __init cpu_psci_cpu_init(unsigned int cpu) { return 0; } diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 2cb0081..98eb68b 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -319,6 +319,23 @@ void __init smp_prepare_boot_cpu(void) } /* + * Initialize cpu operations for a logical cpu and + * set it in the possible mask on success + */ +static int __init smp_cpu_setup(int cpu) +{ + if (cpu_read_ops(cpu)) + return -ENODEV; + + if (cpu_ops[cpu]->cpu_init(cpu)) + return -ENODEV; + + set_cpu_possible(cpu, true); + + return 0; +} + +/* * Enumerate the possible CPU set from the device tree and build the * cpu logical map array containing MPIDR values related to logical * cpus. Assumes that cpu_logical_map(0) has already been initialized. @@ -395,12 +412,6 @@ void __init of_smp_init_cpus(void) if (cpu >= NR_CPUS) goto next; - if (cpu_read_ops(dn, cpu) != 0) - goto next; - - if (cpu_ops[cpu]->cpu_init(dn, cpu)) - goto next; - pr_debug("cpu logical map 0x%llx\n", hwid); cpu_logical_map(cpu) = hwid; next: @@ -418,12 +429,18 @@ next: } /* - * All the cpus that made it to the cpu_logical_map have been - * validated so set them as possible cpus. + * We need to set the cpu_logical_map entries before enabling + * the cpus so that cpu processor description entries (DT cpu nodes + * and ACPI MADT entries) can be retrieved by matching the cpu hwid + * with entries in cpu_logical_map while initializing the cpus. + * If the cpu set-up fails, invalidate the cpu_logical_map entry. */ - for (i = 0; i < NR_CPUS; i++) - if (cpu_logical_map(i) != INVALID_HWID) - set_cpu_possible(i, true); + for (i = 1; i < NR_CPUS; i++) { + if (cpu_logical_map(i) != INVALID_HWID) { + if (smp_cpu_setup(i)) + cpu_logical_map(i) = INVALID_HWID; + } + } } void __init smp_prepare_cpus(unsigned int max_cpus) diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 14944e5..aef3605 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -49,8 +49,14 @@ static void write_pen_release(u64 val) } -static int smp_spin_table_cpu_init(struct device_node *dn, unsigned int cpu) +static int smp_spin_table_cpu_init(unsigned int cpu) { + struct device_node *dn; + + dn = of_get_cpu_node(cpu, NULL); + if (!dn) + return -ENODEV; + /* * Determine the address from which the CPU is polling. */ -- cgit v0.10.2 From 0f0783365cbb7ec13a8f02198f6e1a146d94a5a9 Mon Sep 17 00:00:00 2001 From: Lorenzo Pieralisi Date: Wed, 13 May 2015 14:12:47 +0100 Subject: ARM64: kernel: unify ACPI and DT cpus initialization The code that initializes cpus on arm64 is currently split in two different code paths that carry out DT and ACPI cpus initialization. Most of the code executing SMP initialization is common and should be merged to reduce discrepancies between ACPI and DT initialization and to have code initializing cpus in a single common place in the kernel. This patch refactors arm64 SMP cpus initialization code to merge ACPI and DT boot paths in a common file and to create sanity checks that can be reused by both boot methods. Current code assumes PSCI is the only available boot method when arm64 boots with ACPI; this can be easily extended if/when the ACPI parking protocol is merged into the kernel. Signed-off-by: Sudeep Holla Signed-off-by: Lorenzo Pieralisi Acked-by: Hanjun Guo Acked-by: Mark Rutland Tested-by: Hanjun Guo Tested-by: Mark Rutland [DT] Cc: Will Deacon Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h index 59c05d8..1240b86 100644 --- a/arch/arm64/include/asm/acpi.h +++ b/arch/arm64/include/asm/acpi.h @@ -93,4 +93,8 @@ static inline bool acpi_psci_use_hvc(void) { return false; } static inline void acpi_init_cpus(void) { } #endif /* CONFIG_ACPI */ +static inline const char *acpi_get_enable_method(int cpu) +{ + return acpi_psci_present() ? "psci" : NULL; +} #endif /*_ASM_ACPI_H*/ diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index 7d8b999..8f03446 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -65,7 +65,6 @@ struct cpu_operations { extern const struct cpu_operations *cpu_ops[NR_CPUS]; int __init cpu_read_ops(int cpu); -const struct cpu_operations *cpu_get_ops(const char *name); static inline void __init cpu_read_bootcpu_ops(void) { diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index bf22650..db02be8 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -42,7 +42,7 @@ extern void handle_IPI(int ipinr, struct pt_regs *regs); * Discover the set of possible CPUs and determine their * SMP operations. */ -extern void of_smp_init_cpus(void); +extern void smp_init_cpus(void); /* * Provide a function to raise an IPI cross call on CPUs in callmap. diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index 945cc1f..732f57b 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -36,12 +36,6 @@ EXPORT_SYMBOL(acpi_disabled); int acpi_pci_disabled = 1; /* skip ACPI PCI scan and IRQ initialization */ EXPORT_SYMBOL(acpi_pci_disabled); -/* Processors with enabled flag and sane MPIDR */ -static int enabled_cpus; - -/* Boot CPU is valid or not in MADT */ -static bool bootcpu_valid __initdata; - static bool param_acpi_off __initdata; static bool param_acpi_force __initdata; @@ -95,124 +89,6 @@ void __init __acpi_unmap_table(char *map, unsigned long size) early_memunmap(map, size); } -/** - * acpi_map_gic_cpu_interface - generates a logical cpu number - * and map to MPIDR represented by GICC structure - */ -static void __init -acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) -{ - int i; - u64 mpidr = processor->arm_mpidr & MPIDR_HWID_BITMASK; - bool enabled = !!(processor->flags & ACPI_MADT_ENABLED); - - if (mpidr == INVALID_HWID) { - pr_info("Skip MADT cpu entry with invalid MPIDR\n"); - return; - } - - total_cpus++; - if (!enabled) - return; - - if (enabled_cpus >= NR_CPUS) { - pr_warn("NR_CPUS limit of %d reached, Processor %d/0x%llx ignored.\n", - NR_CPUS, total_cpus, mpidr); - return; - } - - /* Check if GICC structure of boot CPU is available in the MADT */ - if (cpu_logical_map(0) == mpidr) { - if (bootcpu_valid) { - pr_err("Firmware bug, duplicate CPU MPIDR: 0x%llx in MADT\n", - mpidr); - return; - } - - bootcpu_valid = true; - } - - /* - * Duplicate MPIDRs are a recipe for disaster. Scan - * all initialized entries and check for - * duplicates. If any is found just ignore the CPU. - */ - for (i = 1; i < enabled_cpus; i++) { - if (cpu_logical_map(i) == mpidr) { - pr_err("Firmware bug, duplicate CPU MPIDR: 0x%llx in MADT\n", - mpidr); - return; - } - } - - if (!acpi_psci_present()) - return; - - cpu_ops[enabled_cpus] = cpu_get_ops("psci"); - /* CPU 0 was already initialized */ - if (enabled_cpus) { - if (!cpu_ops[enabled_cpus]) - return; - - if (cpu_ops[enabled_cpus]->cpu_init(enabled_cpus)) - return; - - /* map the logical cpu id to cpu MPIDR */ - cpu_logical_map(enabled_cpus) = mpidr; - } - - enabled_cpus++; -} - -static int __init -acpi_parse_gic_cpu_interface(struct acpi_subtable_header *header, - const unsigned long end) -{ - struct acpi_madt_generic_interrupt *processor; - - processor = (struct acpi_madt_generic_interrupt *)header; - - if (BAD_MADT_ENTRY(processor, end)) - return -EINVAL; - - acpi_table_print_madt_entry(header); - acpi_map_gic_cpu_interface(processor); - return 0; -} - -/* Parse GIC cpu interface entries in MADT for SMP init */ -void __init acpi_init_cpus(void) -{ - int count, i; - - /* - * do a partial walk of MADT to determine how many CPUs - * we have including disabled CPUs, and get information - * we need for SMP init - */ - count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, - acpi_parse_gic_cpu_interface, 0); - - if (!count) { - pr_err("No GIC CPU interface entries present\n"); - return; - } else if (count < 0) { - pr_err("Error parsing GIC CPU interface entry\n"); - return; - } - - if (!bootcpu_valid) { - pr_err("MADT missing boot CPU MPIDR, not enabling secondaries\n"); - return; - } - - for (i = 0; i < enabled_cpus; i++) - set_cpu_possible(i, true); - - /* Make boot-up look pretty */ - pr_info("%d CPUs enabled, %d CPUs total\n", enabled_cpus, total_cpus); -} - /* * acpi_fadt_sanity_check() - Check FADT presence and carry out sanity * checks on it diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index bbda2d2..5ea337d 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -16,11 +16,13 @@ * along with this program. If not, see . */ -#include -#include +#include #include #include #include +#include +#include +#include extern const struct cpu_operations smp_spin_table_ops; extern const struct cpu_operations cpu_psci_ops; @@ -35,7 +37,7 @@ static const struct cpu_operations *supported_cpu_ops[] __initconst = { NULL, }; -const struct cpu_operations * __init cpu_get_ops(const char *name) +static const struct cpu_operations * __init cpu_get_ops(const char *name) { const struct cpu_operations **ops = supported_cpu_ops; @@ -49,36 +51,51 @@ const struct cpu_operations * __init cpu_get_ops(const char *name) return NULL; } +static const char *__init cpu_read_enable_method(int cpu) +{ + const char *enable_method; + + if (acpi_disabled) { + struct device_node *dn = of_get_cpu_node(cpu, NULL); + + if (!dn) { + if (!cpu) + pr_err("Failed to find device node for boot cpu\n"); + return NULL; + } + + enable_method = of_get_property(dn, "enable-method", NULL); + if (!enable_method) { + /* + * The boot CPU may not have an enable method (e.g. + * when spin-table is used for secondaries). + * Don't warn spuriously. + */ + if (cpu != 0) + pr_err("%s: missing enable-method property\n", + dn->full_name); + } + } else { + enable_method = acpi_get_enable_method(cpu); + if (!enable_method) + pr_err("Unsupported ACPI enable-method\n"); + } + + return enable_method; +} /* - * Read a cpu's enable method from the device tree and record it in cpu_ops. + * Read a cpu's enable method and record it in cpu_ops. */ int __init cpu_read_ops(int cpu) { - const char *enable_method; - struct device_node *dn = of_get_cpu_node(cpu, NULL); + const char *enable_method = cpu_read_enable_method(cpu); - if (!dn) { - if (!cpu) - pr_err("Failed to find device node for boot cpu\n"); + if (!enable_method) return -ENODEV; - } - - enable_method = of_get_property(dn, "enable-method", NULL); - if (!enable_method) { - /* - * The boot CPU may not have an enable method (e.g. when - * spin-table is used for secondaries). Don't warn spuriously. - */ - if (cpu != 0) - pr_err("%s: missing enable-method property\n", - dn->full_name); - return -ENOENT; - } cpu_ops[cpu] = cpu_get_ops(enable_method); if (!cpu_ops[cpu]) { - pr_warn("%s: unsupported enable-method property: %s\n", - dn->full_name, enable_method); + pr_warn("Unsupported enable-method: %s\n", enable_method); return -EOPNOTSUPP; } diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 7475313..508cca1 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -408,16 +408,13 @@ void __init setup_arch(char **cmdline_p) if (acpi_disabled) { unflatten_device_tree(); psci_dt_init(); - cpu_read_bootcpu_ops(); -#ifdef CONFIG_SMP - of_smp_init_cpus(); -#endif } else { psci_acpi_init(); - acpi_init_cpus(); } + cpu_read_bootcpu_ops(); #ifdef CONFIG_SMP + smp_init_cpus(); smp_build_mpidr_hash(); #endif diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 98eb68b..b698208 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -17,6 +17,7 @@ * along with this program. If not, see . */ +#include #include #include #include @@ -318,6 +319,49 @@ void __init smp_prepare_boot_cpu(void) set_my_cpu_offset(per_cpu_offset(smp_processor_id())); } +static u64 __init of_get_cpu_mpidr(struct device_node *dn) +{ + const __be32 *cell; + u64 hwid; + + /* + * A cpu node with missing "reg" property is + * considered invalid to build a cpu_logical_map + * entry. + */ + cell = of_get_property(dn, "reg", NULL); + if (!cell) { + pr_err("%s: missing reg property\n", dn->full_name); + return INVALID_HWID; + } + + hwid = of_read_number(cell, of_n_addr_cells(dn)); + /* + * Non affinity bits must be set to 0 in the DT + */ + if (hwid & ~MPIDR_HWID_BITMASK) { + pr_err("%s: invalid reg property\n", dn->full_name); + return INVALID_HWID; + } + return hwid; +} + +/* + * Duplicate MPIDRs are a recipe for disaster. Scan all initialized + * entries and check for duplicates. If any is found just ignore the + * cpu. cpu_logical_map was initialized to INVALID_HWID to avoid + * matching valid MPIDR values. + */ +static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid) +{ + unsigned int i; + + for (i = 1; (i < cpu) && (i < NR_CPUS); i++) + if (cpu_logical_map(i) == hwid) + return true; + return false; +} + /* * Initialize cpu operations for a logical cpu and * set it in the possible mask on success @@ -335,57 +379,98 @@ static int __init smp_cpu_setup(int cpu) return 0; } +static bool bootcpu_valid __initdata; +static unsigned int cpu_count = 1; + +#ifdef CONFIG_ACPI +/* + * acpi_map_gic_cpu_interface - parse processor MADT entry + * + * Carry out sanity checks on MADT processor entry and initialize + * cpu_logical_map on success + */ +static void __init +acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) +{ + u64 hwid = processor->arm_mpidr; + + if (hwid & ~MPIDR_HWID_BITMASK || hwid == INVALID_HWID) { + pr_err("skipping CPU entry with invalid MPIDR 0x%llx\n", hwid); + return; + } + + if (!(processor->flags & ACPI_MADT_ENABLED)) { + pr_err("skipping disabled CPU entry with 0x%llx MPIDR\n", hwid); + return; + } + + if (is_mpidr_duplicate(cpu_count, hwid)) { + pr_err("duplicate CPU MPIDR 0x%llx in MADT\n", hwid); + return; + } + + /* Check if GICC structure of boot CPU is available in the MADT */ + if (cpu_logical_map(0) == hwid) { + if (bootcpu_valid) { + pr_err("duplicate boot CPU MPIDR: 0x%llx in MADT\n", + hwid); + return; + } + bootcpu_valid = true; + return; + } + + if (cpu_count >= NR_CPUS) + return; + + /* map the logical cpu id to cpu MPIDR */ + cpu_logical_map(cpu_count) = hwid; + + cpu_count++; +} + +static int __init +acpi_parse_gic_cpu_interface(struct acpi_subtable_header *header, + const unsigned long end) +{ + struct acpi_madt_generic_interrupt *processor; + + processor = (struct acpi_madt_generic_interrupt *)header; + if (BAD_MADT_ENTRY(processor, end)) + return -EINVAL; + + acpi_table_print_madt_entry(header); + + acpi_map_gic_cpu_interface(processor); + + return 0; +} +#else +#define acpi_table_parse_madt(...) do { } while (0) +#endif + /* * Enumerate the possible CPU set from the device tree and build the * cpu logical map array containing MPIDR values related to logical * cpus. Assumes that cpu_logical_map(0) has already been initialized. */ -void __init of_smp_init_cpus(void) +void __init of_parse_and_init_cpus(void) { struct device_node *dn = NULL; - unsigned int i, cpu = 1; - bool bootcpu_valid = false; while ((dn = of_find_node_by_type(dn, "cpu"))) { - const u32 *cell; - u64 hwid; + u64 hwid = of_get_cpu_mpidr(dn); - /* - * A cpu node with missing "reg" property is - * considered invalid to build a cpu_logical_map - * entry. - */ - cell = of_get_property(dn, "reg", NULL); - if (!cell) { - pr_err("%s: missing reg property\n", dn->full_name); + if (hwid == INVALID_HWID) goto next; - } - hwid = of_read_number(cell, of_n_addr_cells(dn)); - /* - * Non affinity bits must be set to 0 in the DT - */ - if (hwid & ~MPIDR_HWID_BITMASK) { - pr_err("%s: invalid reg property\n", dn->full_name); + if (is_mpidr_duplicate(cpu_count, hwid)) { + pr_err("%s: duplicate cpu reg properties in the DT\n", + dn->full_name); goto next; } /* - * Duplicate MPIDRs are a recipe for disaster. Scan - * all initialized entries and check for - * duplicates. If any is found just ignore the cpu. - * cpu_logical_map was initialized to INVALID_HWID to - * avoid matching valid MPIDR values. - */ - for (i = 1; (i < cpu) && (i < NR_CPUS); i++) { - if (cpu_logical_map(i) == hwid) { - pr_err("%s: duplicate cpu reg properties in the DT\n", - dn->full_name); - goto next; - } - } - - /* * The numbering scheme requires that the boot CPU * must be assigned logical id 0. Record it so that * the logical map built from DT is validated and can @@ -409,22 +494,42 @@ void __init of_smp_init_cpus(void) continue; } - if (cpu >= NR_CPUS) + if (cpu_count >= NR_CPUS) goto next; pr_debug("cpu logical map 0x%llx\n", hwid); - cpu_logical_map(cpu) = hwid; + cpu_logical_map(cpu_count) = hwid; next: - cpu++; + cpu_count++; } +} + +/* + * Enumerate the possible CPU set from the device tree or ACPI and build the + * cpu logical map array containing MPIDR values related to logical + * cpus. Assumes that cpu_logical_map(0) has already been initialized. + */ +void __init smp_init_cpus(void) +{ + int i; + + if (acpi_disabled) + of_parse_and_init_cpus(); + else + /* + * do a walk of MADT to determine how many CPUs + * we have including disabled CPUs, and get information + * we need for SMP init + */ + acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, + acpi_parse_gic_cpu_interface, 0); - /* sanity check */ - if (cpu > NR_CPUS) - pr_warning("no. of cores (%d) greater than configured maximum of %d - clipping\n", - cpu, NR_CPUS); + if (cpu_count > NR_CPUS) + pr_warn("no. of cores (%d) greater than configured maximum of %d - clipping\n", + cpu_count, NR_CPUS); if (!bootcpu_valid) { - pr_err("DT missing boot CPU MPIDR, not enabling secondaries\n"); + pr_err("missing boot CPU MPIDR, not enabling secondaries\n"); return; } -- cgit v0.10.2 From 5b09a094f2fb768c76c8d4a82503df6fc7e1df63 Mon Sep 17 00:00:00 2001 From: Hou Pengyang Date: Sun, 10 May 2015 11:07:40 +0000 Subject: arm64: perf: Fix callchain parse error with kernel tracepoint events For ARM64, when tracing with tracepoint events, the IP and pstate are set to 0, preventing the perf code parsing the callchain and resolving the symbols correctly. ./perf record -e sched:sched_switch -g --call-graph dwarf ls [ perf record: Captured and wrote 0.146 MB perf.data ] ./perf report -f Samples: 194 of event 'sched:sched_switch', Event count (approx.): 194 Children Self Command Shared Object Symbol 100.00% 100.00% ls [unknown] [.] 0000000000000000 The fix is to implement perf_arch_fetch_caller_regs for ARM64, which fills several necessary registers used for callchain unwinding, including pc,sp, fp and spsr . With this patch, callchain can be parsed correctly as follows: ...... + 2.63% 0.00% ls [kernel.kallsyms] [k] vfs_symlink + 2.63% 0.00% ls [kernel.kallsyms] [k] follow_down + 2.63% 0.00% ls [kernel.kallsyms] [k] pfkey_get + 2.63% 0.00% ls [kernel.kallsyms] [k] do_execveat_common.isra.33 - 2.63% 0.00% ls [kernel.kallsyms] [k] pfkey_send_policy_notify pfkey_send_policy_notify pfkey_get v9fs_vfs_rename page_follow_link_light link_path_walk el0_svc_naked ....... Signed-off-by: Hou Pengyang Acked-by: Will Deacon Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/perf_event.h b/arch/arm64/include/asm/perf_event.h index d26d1d5..6471773 100644 --- a/arch/arm64/include/asm/perf_event.h +++ b/arch/arm64/include/asm/perf_event.h @@ -24,4 +24,11 @@ extern unsigned long perf_misc_flags(struct pt_regs *regs); #define perf_misc_flags(regs) perf_misc_flags(regs) #endif +#define perf_arch_fetch_caller_regs(regs, __ip) { \ + (regs)->pc = (__ip); \ + (regs)->regs[29] = (unsigned long) __builtin_frame_address(0); \ + (regs)->sp = current_stack_pointer; \ + (regs)->pstate = PSR_MODE_EL1h; \ +} + #endif -- cgit v0.10.2 From 05981277a4de1ad631a4b2c39c4fbc1db69b6f23 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 12 May 2015 14:50:05 -0700 Subject: arm64: Use common outgoing-CPU-notification code This commit removes the open-coded CPU-offline notification with new common code. In particular, this change avoids calling scheduler code using RCU from an offline CPU that RCU is ignoring. This is a minimal change. A more intrusive change might invoke the cpu_check_up_prepare() and cpu_set_state_online() functions at CPU-online time, which would allow onlining throw an error if the CPU did not go offline properly. Signed-off-by: Paul E. McKenney Cc: linux-arm-kernel@lists.infradead.org Acked-by: Will Deacon Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index b698208..65f1a7f 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -254,15 +254,13 @@ static int op_cpu_kill(unsigned int cpu) return cpu_ops[cpu]->cpu_kill(cpu); } -static DECLARE_COMPLETION(cpu_died); - /* * called on the thread which is asking for a CPU to be shutdown - * waits until shutdown has completed, or it is timed out. */ void __cpu_die(unsigned int cpu) { - if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) { + if (!cpu_wait_death(cpu, 5)) { pr_crit("CPU%u: cpu didn't die\n", cpu); return; } @@ -295,7 +293,7 @@ void cpu_die(void) local_irq_disable(); /* Tell __cpu_die() that this CPU is now safe to dispose of */ - complete(&cpu_died); + (void)cpu_report_death(); /* * Actually shutdown the CPU. This must never fail. The specific hotplug -- cgit v0.10.2 From 538b9b25fa5dd372eeb3eabc85cb24d5df9e317b Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Fri, 1 May 2015 10:53:39 +0100 Subject: arm/arm64: kvm: add missing PSCI include We make use of the PSCI function IDs, but don't explicitly include the header which defines them. Relying on transitive header includes is fragile and will be broken as headers are refactored. This patch includes the relevant header file directly so as to avoid future breakage. Signed-off-by: Mark Rutland Acked-by: Christoffer Dall Reviewed-by: Hanjun Guo Tested-by: Hanjun Guo Cc: Marc Zyngier diff --git a/arch/arm/kvm/psci.c b/arch/arm/kvm/psci.c index 02fa8ef..7e9398c 100644 --- a/arch/arm/kvm/psci.c +++ b/arch/arm/kvm/psci.c @@ -24,6 +24,8 @@ #include #include +#include + /* * This is an implementation of the Power State Coordination Interface * as described in ARM document number ARM DEN 0022A. -- cgit v0.10.2 From 6ee3c78cecc795e87de9552baca76ea88292556d Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 22 Apr 2015 17:26:18 +0100 Subject: arm64: smp_plat: add get_logical_index The PSCI MIGRATE_INFO_UP_CPU call returns a physical ID, which we will need to map back to a Linux logical ID. Implement a reusable get_logical_index to map from a physical ID to a logical ID. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Acked-by: Lorenzo Pieralisi Reviewed-by: Hanjun Guo Tested-by: Hanjun Guo Cc: Will Deacon diff --git a/arch/arm64/include/asm/smp_plat.h b/arch/arm64/include/asm/smp_plat.h index 8dcd61e..7abf757 100644 --- a/arch/arm64/include/asm/smp_plat.h +++ b/arch/arm64/include/asm/smp_plat.h @@ -19,6 +19,8 @@ #ifndef __ASM_SMP_PLAT_H #define __ASM_SMP_PLAT_H +#include + #include struct mpidr_hash { @@ -39,6 +41,20 @@ static inline u32 mpidr_hash_size(void) */ extern u64 __cpu_logical_map[NR_CPUS]; #define cpu_logical_map(cpu) __cpu_logical_map[cpu] +/* + * Retrieve logical cpu index corresponding to a given MPIDR.Aff* + * - mpidr: MPIDR.Aff* bits to be used for the look-up + * + * Returns the cpu logical index or -EINVAL on look-up error + */ +static inline int get_logical_index(u64 mpidr) +{ + int cpu; + for (cpu = 0; cpu < nr_cpu_ids; cpu++) + if (cpu_logical_map(cpu) == mpidr) + return cpu; + return -EINVAL; +} void __init do_post_cpus_up_work(void); -- cgit v0.10.2 From 6b99c68cb5dd274d79451e5135f9450f7c01ca52 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 20 Apr 2015 17:55:30 +0100 Subject: arm64: smp: consistently use error codes cpu_kill currently returns one for success and zero for failure, which is unlike all the other cpu_operations, which return zero for success and an error code upon failure. This difference is unnecessarily confusing. Make cpu_kill consistent with the other cpu_operations. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Acked-by: Lorenzo Pieralisi Reviewed-by: Hanjun Guo Tested-by: Hanjun Guo Cc: Will Deacon diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index efe3480..17e717c 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -501,7 +501,7 @@ static int cpu_psci_cpu_kill(unsigned int cpu) int err, i; if (!psci_ops.affinity_info) - return 1; + return 0; /* * cpu_kill could race with cpu_die and we can * potentially end up declaring this cpu undead @@ -512,7 +512,7 @@ static int cpu_psci_cpu_kill(unsigned int cpu) err = psci_ops.affinity_info(cpu_logical_map(cpu), 0); if (err == PSCI_0_2_AFFINITY_LEVEL_OFF) { pr_info("CPU%d killed.\n", cpu); - return 1; + return 0; } msleep(10); @@ -521,8 +521,7 @@ static int cpu_psci_cpu_kill(unsigned int cpu) pr_warn("CPU%d may not have shut down cleanly (AFFINITY_INFO reports %d)\n", cpu, err); - /* Make op_cpu_kill() fail. */ - return 0; + return -ETIMEDOUT; } #endif #endif diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index b698208..2b503a3 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -249,7 +249,7 @@ static int op_cpu_kill(unsigned int cpu) * time and hope that it's dead, so let's skip the wait and just hope. */ if (!cpu_ops[cpu]->cpu_kill) - return 1; + return 0; return cpu_ops[cpu]->cpu_kill(cpu); } @@ -262,6 +262,8 @@ static DECLARE_COMPLETION(cpu_died); */ void __cpu_die(unsigned int cpu) { + int err; + if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) { pr_crit("CPU%u: cpu didn't die\n", cpu); return; @@ -274,8 +276,10 @@ void __cpu_die(unsigned int cpu) * verify that it has really left the kernel before we consider * clobbering anything it might still be using. */ - if (!op_cpu_kill(cpu)) - pr_warn("CPU%d may not have shut down cleanly\n", cpu); + err = op_cpu_kill(cpu); + if (err) + pr_warn("CPU%d may not have shut down cleanly: %d\n", + cpu, err); } /* -- cgit v0.10.2 From 2a7cd0ebfc0a5ac2e692e63871e0ff6a50d5de46 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 22 Apr 2015 16:22:55 +0100 Subject: arm64: psci: remove unnecessary id indirection PSCI 0.1 did not define canonical IDs for CPU_ON, CPU_OFF, CPU_SUSPEND, or MIGRATE, and so these need to be provided when using firmware compliant to PSCI 0.1. However, functions introduced in 0.2 or later have canonical IDs, and these cannot be provided via DT. There's no need to indirect the IDs via a table; they can be used directly at callsites (and already are for SYSTEM_OFF and SYSTEM_RESET). This patch removes the unnecessary function ID indirection for AFFINITY_INFO and MIGRATE_INFO_TYPE. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Acked-by: Lorenzo Pieralisi Reviewed-by: Hanjun Guo Tested-by: Hanjun Guo Cc: Will Deacon diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index 17e717c..0c1efb6 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -67,8 +67,6 @@ enum psci_function { PSCI_FN_CPU_ON, PSCI_FN_CPU_OFF, PSCI_FN_MIGRATE, - PSCI_FN_AFFINITY_INFO, - PSCI_FN_MIGRATE_INFO_TYPE, PSCI_FN_MAX, }; @@ -168,22 +166,13 @@ static int psci_migrate(unsigned long cpuid) static int psci_affinity_info(unsigned long target_affinity, unsigned long lowest_affinity_level) { - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_AFFINITY_INFO]; - err = invoke_psci_fn(fn, target_affinity, lowest_affinity_level, 0); - return err; + return invoke_psci_fn(PSCI_0_2_FN64_AFFINITY_INFO, target_affinity, + lowest_affinity_level, 0); } static int psci_migrate_info_type(void) { - int err; - u32 fn; - - fn = psci_function_id[PSCI_FN_MIGRATE_INFO_TYPE]; - err = invoke_psci_fn(fn, 0, 0, 0); - return err; + return invoke_psci_fn(PSCI_0_2_FN_MIGRATE_INFO_TYPE, 0, 0, 0); } static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) @@ -293,11 +282,8 @@ static void __init psci_0_2_set_functions(void) psci_function_id[PSCI_FN_MIGRATE] = PSCI_0_2_FN64_MIGRATE; psci_ops.migrate = psci_migrate; - psci_function_id[PSCI_FN_AFFINITY_INFO] = PSCI_0_2_FN64_AFFINITY_INFO; psci_ops.affinity_info = psci_affinity_info; - psci_function_id[PSCI_FN_MIGRATE_INFO_TYPE] = - PSCI_0_2_FN_MIGRATE_INFO_TYPE; psci_ops.migrate_info_type = psci_migrate_info_type; arm_pm_restart = psci_sys_reset; -- cgit v0.10.2 From a06eed3e90c272675f2ef50f5bc5b3ec91652d77 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 20 Apr 2015 14:51:00 +0100 Subject: arm64: psci: support unsigned return values PSCI_VERSION and MIGRATE_INFO_TYPE_UP_CPU return unsigned values, with the latter returning a 64-bit value. However, the PSCI invocation functions have prototypes returning int. This patch upgrades the invocation functions to return unsigned long, with a new typedef to keep things legible. As PSCI_VERSION cannot return a negative value, the erroneous check against PSCI_RET_NOT_SUPPORTED is also removed. The unrelated psci_initcall_t typedef is moved closer to its first user, to avoid confusion with the invocation functions. In preparation for sharing the code with ARM, unsigned long is used in preference of u64. In the SMC32 calling convention, the relevant fields will be 32 bits wide. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Reviewed-by: Hanjun Guo Tested-by: Hanjun Guo Cc: Lorenzo Pieralisi Cc: Will Deacon diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index 0c1efb6..7d441b3 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -56,11 +56,11 @@ struct psci_operations { static struct psci_operations psci_ops; -static int (*invoke_psci_fn)(u64, u64, u64, u64); -typedef int (*psci_initcall_t)(const struct device_node *); - -asmlinkage int __invoke_psci_fn_hvc(u64, u64, u64, u64); -asmlinkage int __invoke_psci_fn_smc(u64, u64, u64, u64); +typedef unsigned long (psci_fn)(unsigned long, unsigned long, + unsigned long, unsigned long); +asmlinkage psci_fn __invoke_psci_fn_hvc; +asmlinkage psci_fn __invoke_psci_fn_smc; +static psci_fn *invoke_psci_fn; enum psci_function { PSCI_FN_CPU_SUSPEND, @@ -112,12 +112,9 @@ static void psci_power_state_unpack(u32 power_state, PSCI_0_2_POWER_STATE_AFFL_SHIFT; } -static int psci_get_version(void) +static u32 psci_get_version(void) { - int err; - - err = invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); - return err; + return invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); } static int psci_cpu_suspend(struct psci_power_state state, @@ -296,25 +293,15 @@ static void __init psci_0_2_set_functions(void) */ static int __init psci_probe(void) { - int ver = psci_get_version(); - - if (ver == PSCI_RET_NOT_SUPPORTED) { - /* - * PSCI versions >=0.2 mandates implementation of - * PSCI_VERSION. - */ - pr_err("PSCI firmware does not comply with the v0.2 spec.\n"); - return -EOPNOTSUPP; - } else { - pr_info("PSCIv%d.%d detected in firmware.\n", - PSCI_VERSION_MAJOR(ver), - PSCI_VERSION_MINOR(ver)); - - if (PSCI_VERSION_MAJOR(ver) == 0 && - PSCI_VERSION_MINOR(ver) < 2) { - pr_err("Conflicting PSCI version detected.\n"); - return -EINVAL; - } + u32 ver = psci_get_version(); + + pr_info("PSCIv%d.%d detected in firmware.\n", + PSCI_VERSION_MAJOR(ver), + PSCI_VERSION_MINOR(ver)); + + if (PSCI_VERSION_MAJOR(ver) == 0 && PSCI_VERSION_MINOR(ver) < 2) { + pr_err("Conflicting PSCI version detected.\n"); + return -EINVAL; } psci_0_2_set_functions(); @@ -322,6 +309,8 @@ static int __init psci_probe(void) return 0; } +typedef int (*psci_initcall_t)(const struct device_node *); + /* * PSCI init function for PSCI versions >=0.2 * -- cgit v0.10.2 From ff3010e6fcdb5f7e6999c6026ab7fcf835d54c5a Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 22 Apr 2015 18:10:26 +0100 Subject: arm64: psci: account for Trusted OS instances Software resident in the secure world (a "Trusted OS") may cause CPU_OFF calls for the CPU it is resident on to be denied. Such a denial would be fatal for the kernel, and so we must detect when this can happen before the point of no return. This patch implements Trusted OS detection for PSCI 0.2+ systems, using MIGRATE_INFO_TYPE and MIGRATE_INFO_UP_CPU. When a trusted OS is detected as resident on a particular CPU, attempts to hot unplug that CPU will be denied early, before they can prove fatal. Trusted OS migration is not implemented by this patch. Implementation of migratable UP trusted OSs seems unlikely, and the right policy for migration is unclear (and will likely differ across implementations). As such, it is likely that migration will require cooperation with Trusted OS drivers. PSCI implementations prior to 0.1 do not provide the facility to detect the presence of a Trusted OS, nor the CPU any such OS is resident on, so without additional information it is not possible to handle Trusted OSs with PSCI 0.1. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Tested-by: Hanjun Guo Cc: Lorenzo Pieralisi Cc: Will Deacon diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index 7d441b3..ffa17d1 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -43,6 +44,19 @@ struct psci_power_state { u8 affinity_level; }; +/* + * The CPU any Trusted OS is resident on. The trusted OS may reject CPU_OFF + * calls to its resident CPU, so we must avoid issuing those. We never migrate + * a Trusted OS even if it claims to be capable of migration -- doing so will + * require cooperation with a Trusted OS driver. + */ +static int resident_cpu = -1; + +static bool psci_tos_resident_on(int cpu) +{ + return cpu == resident_cpu; +} + struct psci_operations { int (*cpu_suspend)(struct psci_power_state state, unsigned long entry_point); @@ -172,6 +186,11 @@ static int psci_migrate_info_type(void) return invoke_psci_fn(PSCI_0_2_FN_MIGRATE_INFO_TYPE, 0, 0, 0); } +static unsigned long psci_migrate_info_up_cpu(void) +{ + return invoke_psci_fn(PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU, 0, 0, 0); +} + static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) { int i, ret, count = 0; @@ -264,6 +283,46 @@ static void psci_sys_poweroff(void) invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0); } +/* + * Detect the presence of a resident Trusted OS which may cause CPU_OFF to + * return DENIED (which would be fatal). + */ +static void __init psci_init_migrate(void) +{ + unsigned long cpuid; + int type, cpu; + + type = psci_ops.migrate_info_type(); + + if (type == PSCI_0_2_TOS_MP) { + pr_info("Trusted OS migration not required\n"); + return; + } + + if (type == PSCI_RET_NOT_SUPPORTED) { + pr_info("MIGRATE_INFO_TYPE not supported.\n"); + return; + } + + if (type != PSCI_0_2_TOS_UP_MIGRATE && + type != PSCI_0_2_TOS_UP_NO_MIGRATE) { + pr_err("MIGRATE_INFO_TYPE returned unknown type (%d)\n", type); + return; + } + + cpuid = psci_migrate_info_up_cpu(); + if (cpuid & ~MPIDR_HWID_BITMASK) { + pr_warn("MIGRATE_INFO_UP_CPU reported invalid physical ID (0x%lx)\n", + cpuid); + return; + } + + cpu = get_logical_index(cpuid); + resident_cpu = cpu >= 0 ? cpu : -1; + + pr_info("Trusted OS resident on physical CPU 0x%lx\n", cpuid); +} + static void __init psci_0_2_set_functions(void) { pr_info("Using standard PSCI v0.2 function IDs\n"); @@ -306,6 +365,8 @@ static int __init psci_probe(void) psci_0_2_set_functions(); + psci_init_migrate(); + return 0; } @@ -452,6 +513,11 @@ static int cpu_psci_cpu_disable(unsigned int cpu) /* Fail early if we don't have CPU_OFF support */ if (!psci_ops.cpu_off) return -EOPNOTSUPP; + + /* Trusted OS will deny CPU_OFF */ + if (psci_tos_resident_on(cpu)) + return -EPERM; + return 0; } -- cgit v0.10.2 From c8cc42737788537ebef810ee22400f757e1819ca Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 30 Apr 2015 17:59:03 +0100 Subject: arm64: psci: kill psci_power_state A PSCI 1.0 implementation may choose to use the new extended StateID format, the presence of which may be queried via the PSCI_FEATURES call. The layout of this new StateID format is incompatible with the existing format, and so to handle both we must abstract attempts to parse the fields. In preparation for PSCI 1.0 support, this patch introduces psci_power_state_loses_context and psci_power_state_is_valid functions to query information from a PSCI power state, which is no longer decomposed (and hence the pack/unpack functions are removed). As it is no longer decomposed, it is now passed round as an opaque u32 token. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Acked-by: Lorenzo Pieralisi Tested-by: Hanjun Guo Cc: Will Deacon diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index ffa17d1..0e2a440 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -38,11 +38,19 @@ #define PSCI_POWER_STATE_TYPE_STANDBY 0 #define PSCI_POWER_STATE_TYPE_POWER_DOWN 1 -struct psci_power_state { - u16 id; - u8 type; - u8 affinity_level; -}; +static bool psci_power_state_loses_context(u32 state) +{ + return state & PSCI_0_2_POWER_STATE_TYPE_MASK; +} + +static bool psci_power_state_is_valid(u32 state) +{ + const u32 valid_mask = PSCI_0_2_POWER_STATE_ID_MASK | + PSCI_0_2_POWER_STATE_TYPE_MASK | + PSCI_0_2_POWER_STATE_AFFL_MASK; + + return !(state & ~valid_mask); +} /* * The CPU any Trusted OS is resident on. The trusted OS may reject CPU_OFF @@ -58,9 +66,8 @@ static bool psci_tos_resident_on(int cpu) } struct psci_operations { - int (*cpu_suspend)(struct psci_power_state state, - unsigned long entry_point); - int (*cpu_off)(struct psci_power_state state); + int (*cpu_suspend)(u32 state, unsigned long entry_point); + int (*cpu_off)(u32 state); int (*cpu_on)(unsigned long cpuid, unsigned long entry_point); int (*migrate)(unsigned long cpuid); int (*affinity_info)(unsigned long target_affinity, @@ -84,7 +91,7 @@ enum psci_function { PSCI_FN_MAX, }; -static DEFINE_PER_CPU_READ_MOSTLY(struct psci_power_state *, psci_power_state); +static DEFINE_PER_CPU_READ_MOSTLY(u32 *, psci_power_state); static u32 psci_function_id[PSCI_FN_MAX]; @@ -104,53 +111,28 @@ static int psci_to_linux_errno(int errno) return -EINVAL; } -static u32 psci_power_state_pack(struct psci_power_state state) -{ - return ((state.id << PSCI_0_2_POWER_STATE_ID_SHIFT) - & PSCI_0_2_POWER_STATE_ID_MASK) | - ((state.type << PSCI_0_2_POWER_STATE_TYPE_SHIFT) - & PSCI_0_2_POWER_STATE_TYPE_MASK) | - ((state.affinity_level << PSCI_0_2_POWER_STATE_AFFL_SHIFT) - & PSCI_0_2_POWER_STATE_AFFL_MASK); -} - -static void psci_power_state_unpack(u32 power_state, - struct psci_power_state *state) -{ - state->id = (power_state & PSCI_0_2_POWER_STATE_ID_MASK) >> - PSCI_0_2_POWER_STATE_ID_SHIFT; - state->type = (power_state & PSCI_0_2_POWER_STATE_TYPE_MASK) >> - PSCI_0_2_POWER_STATE_TYPE_SHIFT; - state->affinity_level = - (power_state & PSCI_0_2_POWER_STATE_AFFL_MASK) >> - PSCI_0_2_POWER_STATE_AFFL_SHIFT; -} - static u32 psci_get_version(void) { return invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0); } -static int psci_cpu_suspend(struct psci_power_state state, - unsigned long entry_point) +static int psci_cpu_suspend(u32 state, unsigned long entry_point) { int err; - u32 fn, power_state; + u32 fn; fn = psci_function_id[PSCI_FN_CPU_SUSPEND]; - power_state = psci_power_state_pack(state); - err = invoke_psci_fn(fn, power_state, entry_point, 0); + err = invoke_psci_fn(fn, state, entry_point, 0); return psci_to_linux_errno(err); } -static int psci_cpu_off(struct psci_power_state state) +static int psci_cpu_off(u32 state) { int err; - u32 fn, power_state; + u32 fn; fn = psci_function_id[PSCI_FN_CPU_OFF]; - power_state = psci_power_state_pack(state); - err = invoke_psci_fn(fn, power_state, 0, 0); + err = invoke_psci_fn(fn, state, 0, 0); return psci_to_linux_errno(err); } @@ -194,7 +176,7 @@ static unsigned long psci_migrate_info_up_cpu(void) static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) { int i, ret, count = 0; - struct psci_power_state *psci_states; + u32 *psci_states; struct device_node *state_node, *cpu_node; cpu_node = of_get_cpu_node(cpu, NULL); @@ -223,13 +205,13 @@ static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) return -ENOMEM; for (i = 0; i < count; i++) { - u32 psci_power_state; + u32 state; state_node = of_parse_phandle(cpu_node, "cpu-idle-states", i); ret = of_property_read_u32(state_node, "arm,psci-suspend-param", - &psci_power_state); + &state); if (ret) { pr_warn(" * %s missing arm,psci-suspend-param property\n", state_node->full_name); @@ -238,9 +220,13 @@ static int __maybe_unused cpu_psci_cpu_init_idle(unsigned int cpu) } of_node_put(state_node); - pr_debug("psci-power-state %#x index %d\n", psci_power_state, - i); - psci_power_state_unpack(psci_power_state, &psci_states[i]); + pr_debug("psci-power-state %#x index %d\n", state, i); + if (!psci_power_state_is_valid(state)) { + pr_warn("Invalid PSCI power state %#x\n", state); + ret = -EINVAL; + goto free_mem; + } + psci_states[i] = state; } /* Idle states parsed correctly, initialize per-cpu pointer */ per_cpu(psci_power_state, cpu) = psci_states; @@ -528,9 +514,8 @@ static void cpu_psci_cpu_die(unsigned int cpu) * There are no known implementations of PSCI actually using the * power state field, pass a sensible default for now. */ - struct psci_power_state state = { - .type = PSCI_POWER_STATE_TYPE_POWER_DOWN, - }; + u32 state = PSCI_POWER_STATE_TYPE_POWER_DOWN << + PSCI_0_2_POWER_STATE_TYPE_SHIFT; ret = psci_ops.cpu_off(state); @@ -569,7 +554,7 @@ static int cpu_psci_cpu_kill(unsigned int cpu) static int psci_suspend_finisher(unsigned long index) { - struct psci_power_state *state = __this_cpu_read(psci_power_state); + u32 *state = __this_cpu_read(psci_power_state); return psci_ops.cpu_suspend(state[index - 1], virt_to_phys(cpu_resume)); @@ -578,7 +563,7 @@ static int psci_suspend_finisher(unsigned long index) static int __maybe_unused cpu_psci_cpu_suspend(unsigned long index) { int ret; - struct psci_power_state *state = __this_cpu_read(psci_power_state); + u32 *state = __this_cpu_read(psci_power_state); /* * idle state index 0 corresponds to wfi, should never be called * from the cpu_suspend operations @@ -586,7 +571,7 @@ static int __maybe_unused cpu_psci_cpu_suspend(unsigned long index) if (WARN_ON_ONCE(!index)) return -EINVAL; - if (state[index - 1].type == PSCI_POWER_STATE_TYPE_STANDBY) + if (!psci_power_state_loses_context(state[index - 1])) ret = psci_ops.cpu_suspend(state[index - 1], 0); else ret = __cpu_suspend(index, psci_suspend_finisher); -- cgit v0.10.2 From c5a1330573c1748179898f4799f130e416ce4738 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Thu, 30 Apr 2015 14:22:04 +0100 Subject: arm64: psci: remove ACPI coupling The 32-bit ARM port doesn't have ACPI headers, and conditionally including them is going to look horrendous. In preparation for sharing the PSCI invocation code with 32-bit, move the acpi_psci_* function declarations and definitions such that the PSCI client code need not include ACPI headers. While it would seem like we could simply hide the ACPI includes in psci.h, the ACPI headers have hilarious circular dependencies which make this infeasible without reorganising most of ACPICA. So rather than doing that, move the acpi_psci_* prototypes into psci.h. The psci_acpi_init function is made dependent on CONFIG_ACPI (with a stub implementation in asm/psci.h) such that it need not be built for 32-bit ARM or kernels without ACPI support. The currently missing __init annotations are added to the prototypes in the header. Signed-off-by: Mark Rutland Acked-by: Catalin Marinas Acked-by: Hanjun Guo Reviewed-by: Al Stone Reviewed-by: Ashwin Chaugule Tested-by: Hanjun Guo Cc: Lorenzo Pieralisi Cc: Will Deacon diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h index 1240b86..39248d3 100644 --- a/arch/arm64/include/asm/acpi.h +++ b/arch/arm64/include/asm/acpi.h @@ -16,6 +16,7 @@ #include #include +#include #include /* Basic configuration for ACPI */ @@ -39,18 +40,6 @@ extern int acpi_disabled; extern int acpi_noirq; extern int acpi_pci_disabled; -/* 1 to indicate PSCI 0.2+ is implemented */ -static inline bool acpi_psci_present(void) -{ - return acpi_gbl_FADT.arm_boot_flags & ACPI_FADT_PSCI_COMPLIANT; -} - -/* 1 to indicate HVC must be used instead of SMC as the PSCI conduit */ -static inline bool acpi_psci_use_hvc(void) -{ - return acpi_gbl_FADT.arm_boot_flags & ACPI_FADT_PSCI_USE_HVC; -} - static inline void disable_acpi(void) { acpi_disabled = 1; @@ -88,8 +77,6 @@ static inline void arch_fix_phys_package_id(int num, u32 slot) { } void __init acpi_init_cpus(void); #else -static inline bool acpi_psci_present(void) { return false; } -static inline bool acpi_psci_use_hvc(void) { return false; } static inline void acpi_init_cpus(void) { } #endif /* CONFIG_ACPI */ diff --git a/arch/arm64/include/asm/psci.h b/arch/arm64/include/asm/psci.h index 2454bc5..49d7e1a 100644 --- a/arch/arm64/include/asm/psci.h +++ b/arch/arm64/include/asm/psci.h @@ -14,7 +14,15 @@ #ifndef __ASM_PSCI_H #define __ASM_PSCI_H -int psci_dt_init(void); -int psci_acpi_init(void); +int __init psci_dt_init(void); + +#ifdef CONFIG_ACPI +int __init psci_acpi_init(void); +bool __init acpi_psci_present(void); +bool __init acpi_psci_use_hvc(void); +#else +static inline int psci_acpi_init(void) { return 0; } +static inline bool acpi_psci_present(void) { return false; } +#endif #endif /* __ASM_PSCI_H */ diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index 732f57b..19de753 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -89,6 +89,17 @@ void __init __acpi_unmap_table(char *map, unsigned long size) early_memunmap(map, size); } +bool __init acpi_psci_present(void) +{ + return acpi_gbl_FADT.arm_boot_flags & ACPI_FADT_PSCI_COMPLIANT; +} + +/* Whether HVC must be used instead of SMC as the PSCI conduit */ +bool __init acpi_psci_use_hvc(void) +{ + return acpi_gbl_FADT.arm_boot_flags & ACPI_FADT_PSCI_USE_HVC; +} + /* * acpi_fadt_sanity_check() - Check FADT presence and carry out sanity * checks on it diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index 0e2a440..cba05d9 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -15,7 +15,6 @@ #define pr_fmt(fmt) "psci: " fmt -#include #include #include #include @@ -25,7 +24,6 @@ #include #include -#include #include #include #include @@ -446,6 +444,7 @@ int __init psci_dt_init(void) return init_fn(np); } +#ifdef CONFIG_ACPI /* * We use PSCI 0.2+ when ACPI is deployed on ARM64 and it's * explicitly clarified in SBBR @@ -466,6 +465,7 @@ int __init psci_acpi_init(void) return psci_probe(); } +#endif #ifdef CONFIG_SMP -- cgit v0.10.2 From d00a3810c16207d2541b7796a73cca5a24ea3742 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 May 2015 15:39:40 +0100 Subject: arm64: context-switch user tls register tpidr_el0 for compat tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit a4780adeefd0 ("ARM: 7735/2: Preserve the user r/w register TPIDRURW on context switch and fork"), arch/arm/ has context switched the user-writable TLS register, so do the same for compat tasks running under the arm64 kernel. Reported-by: André Hentschel Tested-by: André Hentschel Signed-off-by: Will Deacon Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index d2c37a1..e4c893e 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -78,13 +78,30 @@ struct cpu_context { struct thread_struct { struct cpu_context cpu_context; /* cpu context */ - unsigned long tp_value; + unsigned long tp_value; /* TLS register */ +#ifdef CONFIG_COMPAT + unsigned long tp2_value; +#endif struct fpsimd_state fpsimd_state; unsigned long fault_address; /* fault info */ unsigned long fault_code; /* ESR_EL1 value */ struct debug_info debug; /* debugging */ }; +#ifdef CONFIG_COMPAT +#define task_user_tls(t) \ +({ \ + unsigned long *__tls; \ + if (is_compat_thread(task_thread_info(t))) \ + __tls = &(t)->thread.tp2_value; \ + else \ + __tls = &(t)->thread.tp_value; \ + __tls; \ + }) +#else +#define task_user_tls(t) (&(t)->thread.tp_value) +#endif + #define INIT_THREAD { } static inline void start_thread_common(struct pt_regs *regs, unsigned long pc) diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index c506bee..369f485 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -244,35 +244,35 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, unsigned long stk_sz, struct task_struct *p) { struct pt_regs *childregs = task_pt_regs(p); - unsigned long tls = p->thread.tp_value; memset(&p->thread.cpu_context, 0, sizeof(struct cpu_context)); if (likely(!(p->flags & PF_KTHREAD))) { *childregs = *current_pt_regs(); childregs->regs[0] = 0; - if (is_compat_thread(task_thread_info(p))) { - if (stack_start) + + /* + * Read the current TLS pointer from tpidr_el0 as it may be + * out-of-sync with the saved value. + */ + asm("mrs %0, tpidr_el0" : "=r" (*task_user_tls(p))); + + if (stack_start) { + if (is_compat_thread(task_thread_info(p))) childregs->compat_sp = stack_start; - } else { - /* - * Read the current TLS pointer from tpidr_el0 as it may be - * out-of-sync with the saved value. - */ - asm("mrs %0, tpidr_el0" : "=r" (tls)); - if (stack_start) { - /* 16-byte aligned stack mandatory on AArch64 */ - if (stack_start & 15) - return -EINVAL; + /* 16-byte aligned stack mandatory on AArch64 */ + else if (stack_start & 15) + return -EINVAL; + else childregs->sp = stack_start; - } } + /* * If a TLS pointer was passed to clone (4th argument), use it * for the new thread. */ if (clone_flags & CLONE_SETTLS) - tls = childregs->regs[3]; + p->thread.tp_value = childregs->regs[3]; } else { memset(childregs, 0, sizeof(struct pt_regs)); childregs->pstate = PSR_MODE_EL1h; @@ -281,7 +281,6 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, } p->thread.cpu_context.pc = (unsigned long)ret_from_fork; p->thread.cpu_context.sp = (unsigned long)childregs; - p->thread.tp_value = tls; ptrace_hw_copy_thread(p); @@ -292,18 +291,12 @@ static void tls_thread_switch(struct task_struct *next) { unsigned long tpidr, tpidrro; - if (!is_compat_task()) { - asm("mrs %0, tpidr_el0" : "=r" (tpidr)); - current->thread.tp_value = tpidr; - } + asm("mrs %0, tpidr_el0" : "=r" (tpidr)); + *task_user_tls(current) = tpidr; - if (is_compat_thread(task_thread_info(next))) { - tpidr = 0; - tpidrro = next->thread.tp_value; - } else { - tpidr = next->thread.tp_value; - tpidrro = 0; - } + tpidr = *task_user_tls(next); + tpidrro = is_compat_thread(task_thread_info(next)) ? + next->thread.tp_value : 0; asm( " msr tpidr_el0, %0\n" -- cgit v0.10.2 From 24bbd929e6b9e62afd263c42b4318d3b603c956c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 1 Jun 2015 13:40:31 +0200 Subject: of/fdt: split off FDT self reservation from memreserve processing This splits off the reservation of the memory occupied by the FDT binary itself from the processing of the memory reservations it contains. This is necessary because the physical address of the FDT, which is needed to perform the reservation, may not be known to the FDT driver core, i.e., it may be mapped outside the linear direct mapping, in which case __pa() returns a bogus value. Cc: Russell King Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Acked-by: Rob Herring Acked-by: Mark Rutland Acked-by: Catalin Marinas Signed-off-by: Ard Biesheuvel Signed-off-by: Catalin Marinas diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index be92fa0..8a63b4c 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -268,6 +268,7 @@ void __init arm_memblock_init(const struct machine_desc *mdesc) if (mdesc->reserve) mdesc->reserve(); + early_init_fdt_reserve_self(); early_init_fdt_scan_reserved_mem(); /* reserve memory for DMA contiguous allocations */ diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 597831b..89a05f4 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -170,6 +170,7 @@ void __init arm64_memblock_init(void) memblock_reserve(__virt_to_phys(initrd_start), initrd_end - initrd_start); #endif + early_init_fdt_reserve_self(); early_init_fdt_scan_reserved_mem(); /* 4GB maximum for 32-bit only capable devices */ diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 308c5e1..51ea36f 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -573,6 +573,7 @@ static void __init early_reserve_mem_dt(void) int len; const __be32 *prop; + early_init_fdt_reserve_self(); early_init_fdt_scan_reserved_mem(); dt_root = of_get_flat_dt_root(); diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c index cde35c5d01..f2dd23a 100644 --- a/drivers/of/fdt.c +++ b/drivers/of/fdt.c @@ -580,11 +580,6 @@ void __init early_init_fdt_scan_reserved_mem(void) if (!initial_boot_params) return; - /* Reserve the dtb region */ - early_init_dt_reserve_memory_arch(__pa(initial_boot_params), - fdt_totalsize(initial_boot_params), - 0); - /* Process header /memreserve/ fields */ for (n = 0; ; n++) { fdt_get_mem_rsv(initial_boot_params, n, &base, &size); @@ -598,6 +593,20 @@ void __init early_init_fdt_scan_reserved_mem(void) } /** + * early_init_fdt_reserve_self() - reserve the memory used by the FDT blob + */ +void __init early_init_fdt_reserve_self(void) +{ + if (!initial_boot_params) + return; + + /* Reserve the dtb region */ + early_init_dt_reserve_memory_arch(__pa(initial_boot_params), + fdt_totalsize(initial_boot_params), + 0); +} + +/** * of_scan_flat_dt - scan flattened tree blob and call callback on each. * @it: callback function * @data: context data pointer diff --git a/include/linux/of_fdt.h b/include/linux/of_fdt.h index 587ee50..fd627a5 100644 --- a/include/linux/of_fdt.h +++ b/include/linux/of_fdt.h @@ -64,6 +64,7 @@ extern int early_init_dt_scan_chosen(unsigned long node, const char *uname, extern int early_init_dt_scan_memory(unsigned long node, const char *uname, int depth, void *data); extern void early_init_fdt_scan_reserved_mem(void); +extern void early_init_fdt_reserve_self(void); extern void early_init_dt_add_memory_arch(u64 base, u64 size); extern int early_init_dt_reserve_memory_arch(phys_addr_t base, phys_addr_t size, bool no_map); @@ -91,6 +92,7 @@ extern u64 fdt_translate_address(const void *blob, int node_offset); extern void of_fdt_limit_memory(int limit); #else /* CONFIG_OF_FLATTREE */ static inline void early_init_fdt_scan_reserved_mem(void) {} +static inline void early_init_fdt_reserve_self(void) {} static inline const char *of_flat_dt_get_machine_name(void) { return NULL; } static inline void unflatten_device_tree(void) {} static inline void unflatten_and_copy_device_tree(void) {} -- cgit v0.10.2 From 61bd93ce801bb6df36eda257a9d2d16c02863cdd Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 1 Jun 2015 13:40:32 +0200 Subject: arm64: use fixmap region for permanent FDT mapping Currently, the FDT blob needs to be in the same 512 MB region as the kernel, so that it can be mapped into the kernel virtual memory space very early on using a minimal set of statically allocated translation tables. Now that we have early fixmap support, we can relax this restriction, by moving the permanent FDT mapping to the fixmap region instead. This way, the FDT blob may be anywhere in memory. This also moves the vetting of the FDT to mmu.c, since the early init code in head.S does not handle mapping of the FDT anymore. At the same time, fix up some comments in head.S that have gone stale. Reviewed-by: Mark Rutland Tested-by: Mark Rutland Signed-off-by: Ard Biesheuvel Signed-off-by: Catalin Marinas diff --git a/Documentation/arm64/booting.txt b/Documentation/arm64/booting.txt index f3c05b5..1690350 100644 --- a/Documentation/arm64/booting.txt +++ b/Documentation/arm64/booting.txt @@ -45,11 +45,13 @@ sees fit.) Requirement: MANDATORY -The device tree blob (dtb) must be placed on an 8-byte boundary within -the first 512 megabytes from the start of the kernel image and must not -cross a 2-megabyte boundary. This is to allow the kernel to map the -blob using a single section mapping in the initial page tables. +The device tree blob (dtb) must be placed on an 8-byte boundary and must +not exceed 2 megabytes in size. Since the dtb will be mapped cacheable +using blocks of up to 2 megabytes in size, it must not be placed within +any 2M region which must be mapped with any specific attributes. +NOTE: versions prior to v4.2 also require that the DTB be placed within +the 512 MB region starting at text_offset bytes below the kernel Image. 3. Decompress the kernel image ------------------------------ diff --git a/arch/arm64/include/asm/boot.h b/arch/arm64/include/asm/boot.h new file mode 100644 index 0000000..81151b6 --- /dev/null +++ b/arch/arm64/include/asm/boot.h @@ -0,0 +1,14 @@ + +#ifndef __ASM_BOOT_H +#define __ASM_BOOT_H + +#include + +/* + * arm64 requires the DTB to be 8 byte aligned and + * not exceed 2MB in size. + */ +#define MIN_FDT_ALIGN 8 +#define MAX_FDT_SIZE SZ_2M + +#endif diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 95e6b6d..c0739187 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -17,6 +17,7 @@ #ifndef __ASSEMBLY__ #include +#include #include /* @@ -32,6 +33,20 @@ */ enum fixed_addresses { FIX_HOLE, + + /* + * Reserve a virtual window for the FDT that is 2 MB larger than the + * maximum supported size, and put it at the top of the fixmap region. + * The additional space ensures that any FDT that does not exceed + * MAX_FDT_SIZE can be mapped regardless of whether it crosses any + * 2 MB alignment boundaries. + * + * Keep this at the top so it remains 2 MB aligned. + */ +#define FIX_FDT_SIZE (MAX_FDT_SIZE + SZ_2M) + FIX_FDT_END, + FIX_FDT = FIX_FDT_END + FIX_FDT_SIZE / PAGE_SIZE - 1, + FIX_EARLYCON_MEM_BASE, FIX_TEXT_POKE0, __end_of_permanent_fixed_addresses, diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index 3d31176..79fcfb0 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -34,5 +34,6 @@ extern void init_mem_pgprot(void); extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, unsigned long virt, phys_addr_t size, pgprot_t prot); +extern void *fixmap_remap_fdt(phys_addr_t dt_phys); #endif diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 19f915e..30cffc5 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -237,8 +237,6 @@ ENTRY(stext) bl el2_setup // Drop to EL1, w20=cpu_boot_mode adrp x24, __PHYS_OFFSET bl set_cpu_boot_mode_flag - - bl __vet_fdt bl __create_page_tables // x25=TTBR0, x26=TTBR1 /* * The following calls CPU setup code, see arch/arm64/mm/proc.S for @@ -270,24 +268,6 @@ preserve_boot_args: ENDPROC(preserve_boot_args) /* - * Determine validity of the x21 FDT pointer. - * The dtb must be 8-byte aligned and live in the first 512M of memory. - */ -__vet_fdt: - tst x21, #0x7 - b.ne 1f - cmp x21, x24 - b.lt 1f - mov x0, #(1 << 29) - add x0, x0, x24 - cmp x21, x0 - b.ge 1f - ret -1: - mov x21, #0 - ret -ENDPROC(__vet_fdt) -/* * Macro to create a table entry to the next page. * * tbl: page table address @@ -348,8 +328,7 @@ ENDPROC(__vet_fdt) * required to get the kernel running. The following sections are required: * - identity mapping to enable the MMU (low address, TTBR0) * - first few MB of the kernel linear mapping to jump to once the MMU has - * been enabled, including the FDT blob (TTBR1) - * - pgd entry for fixed mappings (TTBR1) + * been enabled */ __create_page_tables: adrp x25, idmap_pg_dir @@ -439,22 +418,6 @@ __create_page_tables: create_block_map x0, x7, x3, x5, x6 /* - * Map the FDT blob (maximum 2MB; must be within 512MB of - * PHYS_OFFSET). - */ - mov x3, x21 // FDT phys address - and x3, x3, #~((1 << 21) - 1) // 2MB aligned - mov x6, #PAGE_OFFSET - sub x5, x3, x24 // subtract PHYS_OFFSET - tst x5, #~((1 << 29) - 1) // within 512MB? - csel x21, xzr, x21, ne // zero the FDT pointer - b.ne 1f - add x5, x5, x6 // __va(FDT blob) - add x6, x5, #1 << 21 // 2MB for the FDT blob - sub x6, x6, #1 // inclusive range - create_block_map x0, x7, x3, x5, x6 -1: - /* * Since the page tables have been populated with non-cacheable * accesses (MMU disabled), invalidate the idmap and swapper page * tables again to remove any speculatively loaded cache lines. diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 508cca1..ffd3970 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -105,18 +105,6 @@ static struct resource mem_res[] = { #define kernel_code mem_res[0] #define kernel_data mem_res[1] -void __init early_print(const char *str, ...) -{ - char buf[256]; - va_list ap; - - va_start(ap, str); - vsnprintf(buf, sizeof(buf), str, ap); - va_end(ap); - - printk("%s", buf); -} - /* * The recorded values of x0 .. x3 upon kernel entry. */ @@ -326,12 +314,14 @@ static void __init setup_processor(void) static void __init setup_machine_fdt(phys_addr_t dt_phys) { - if (!dt_phys || !early_init_dt_scan(phys_to_virt(dt_phys))) { - early_print("\n" - "Error: invalid device tree blob at physical address 0x%p (virtual address 0x%p)\n" - "The dtb must be 8-byte aligned and passed in the first 512MB of memory\n" - "\nPlease check your bootloader.\n", - dt_phys, phys_to_virt(dt_phys)); + void *dt_virt = fixmap_remap_fdt(dt_phys); + + if (!dt_virt || !early_init_dt_scan(dt_virt)) { + pr_crit("\n" + "Error: invalid device tree blob at physical address %pa (virtual address 0x%p)\n" + "The dtb must be 8-byte aligned and must not exceed 2 MB in size\n" + "\nPlease check your bootloader.", + &dt_phys, dt_virt); while (true) cpu_relax(); @@ -374,8 +364,6 @@ void __init setup_arch(char **cmdline_p) { setup_processor(); - setup_machine_fdt(__fdt_pointer); - init_mm.start_code = (unsigned long) _text; init_mm.end_code = (unsigned long) _etext; init_mm.end_data = (unsigned long) _edata; @@ -386,6 +374,8 @@ void __init setup_arch(char **cmdline_p) early_fixmap_init(); early_ioremap_init(); + setup_machine_fdt(__fdt_pointer); + parse_early_param(); /* diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index 773d37a..9d84feb 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -4,3 +4,5 @@ obj-y := dma-mapping.o extable.o fault.o init.o \ context.o proc.o pageattr.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_ARM64_PTDUMP) += dump.o + +CFLAGS_mmu.o := -I$(srctree)/scripts/dtc/libfdt/ diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 89a05f4..597831b 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -170,7 +170,6 @@ void __init arm64_memblock_init(void) memblock_reserve(__virt_to_phys(initrd_start), initrd_end - initrd_start); #endif - early_init_fdt_reserve_self(); early_init_fdt_scan_reserved_mem(); /* 4GB maximum for 32-bit only capable devices */ diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 5b8b664..82d3435 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -643,3 +644,68 @@ void __set_fixmap(enum fixed_addresses idx, flush_tlb_kernel_range(addr, addr+PAGE_SIZE); } } + +void *__init fixmap_remap_fdt(phys_addr_t dt_phys) +{ + const u64 dt_virt_base = __fix_to_virt(FIX_FDT); + pgprot_t prot = PAGE_KERNEL | PTE_RDONLY; + int granularity, size, offset; + void *dt_virt; + + /* + * Check whether the physical FDT address is set and meets the minimum + * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be + * at least 8 bytes so that we can always access the size field of the + * FDT header after mapping the first chunk, double check here if that + * is indeed the case. + */ + BUILD_BUG_ON(MIN_FDT_ALIGN < 8); + if (!dt_phys || dt_phys % MIN_FDT_ALIGN) + return NULL; + + /* + * Make sure that the FDT region can be mapped without the need to + * allocate additional translation table pages, so that it is safe + * to call create_mapping() this early. + * + * On 64k pages, the FDT will be mapped using PTEs, so we need to + * be in the same PMD as the rest of the fixmap. + * On 4k pages, we'll use section mappings for the FDT so we only + * have to be in the same PUD. + */ + BUILD_BUG_ON(dt_virt_base % SZ_2M); + + if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) { + BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> PMD_SHIFT != + __fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT); + + granularity = PAGE_SIZE; + } else { + BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> PUD_SHIFT != + __fix_to_virt(FIX_BTMAP_BEGIN) >> PUD_SHIFT); + + granularity = PMD_SIZE; + } + + offset = dt_phys % granularity; + dt_virt = (void *)dt_virt_base + offset; + + /* map the first chunk so we can read the size from the header */ + create_mapping(round_down(dt_phys, granularity), dt_virt_base, + granularity, prot); + + if (fdt_check_header(dt_virt) != 0) + return NULL; + + size = fdt_totalsize(dt_virt); + if (size > MAX_FDT_SIZE) + return NULL; + + if (offset + size > granularity) + create_mapping(round_down(dt_phys, granularity), dt_virt_base, + round_up(offset + size, granularity), prot); + + memblock_reserve(dt_phys, size); + + return dt_virt; +} -- cgit v0.10.2 From 5dfe9d7d23c26d029415379630523f141a748c5b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 1 Jun 2015 13:40:33 +0200 Subject: arm64: reduce ID map to a single page Commit ea8c2e112445 ("arm64: Extend the idmap to the whole kernel image") changed the early page table code so that the entire kernel Image is covered by the identity map. This allows functions that need to enable or disable the MMU to reside anywhere in the kernel Image. However, this change has the unfortunate side effect that the Image cannot cross a physical 512 MB alignment boundary anymore, since the early page table code cannot deal with the Image crossing a /virtual/ 512 MB alignment boundary. So instead, reduce the ID map to a single page, that is populated by the contents of the .idmap.text section. Only three functions reside there at the moment: __enable_mmu(), cpu_resume_mmu() and cpu_reset(). If new code is introduced that needs to manipulate the MMU state, it should be added to this section as well. Reviewed-by: Mark Rutland Tested-by: Mark Rutland Signed-off-by: Ard Biesheuvel Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 30cffc5..c0ff3ce 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -361,7 +361,7 @@ __create_page_tables: * Create the identity mapping. */ mov x0, x25 // idmap_pg_dir - adrp x3, KERNEL_START // __pa(KERNEL_START) + adrp x3, __idmap_text_start // __pa(__idmap_text_start) #ifndef CONFIG_ARM64_VA_BITS_48 #define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3) @@ -384,11 +384,11 @@ __create_page_tables: /* * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the - * entire kernel image can be ID mapped. As T0SZ == (64 - #bits used), + * entire ID map region can be mapped. As T0SZ == (64 - #bits used), * this number conveniently equals the number of leading zeroes in - * the physical address of KERNEL_END. + * the physical address of __idmap_text_end. */ - adrp x5, KERNEL_END + adrp x5, __idmap_text_end clz x5, x5 cmp x5, TCR_T0SZ(VA_BITS) // default T0SZ small enough? b.ge 1f // .. then skip additional level @@ -403,8 +403,8 @@ __create_page_tables: #endif create_pgd_entry x0, x3, x5, x6 - mov x5, x3 // __pa(KERNEL_START) - adr_l x6, KERNEL_END // __pa(KERNEL_END) + mov x5, x3 // __pa(__idmap_text_start) + adr_l x6, __idmap_text_end // __pa(__idmap_text_end) create_block_map x0, x7, x3, x5, x6 /* @@ -632,6 +632,7 @@ ENDPROC(__secondary_switched) * * other registers depend on the function called upon completion */ + .section ".idmap.text", "ax" __enable_mmu: ldr x5, =vectors msr vbar_el1, x5 diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index ede186c..811e61a 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -130,12 +130,14 @@ ENDPROC(__cpu_suspend_enter) /* * x0 must contain the sctlr value retrieved from restored context */ + .pushsection ".idmap.text", "ax" ENTRY(cpu_resume_mmu) ldr x3, =cpu_resume_after_mmu msr sctlr_el1, x0 // restore sctlr_el1 isb br x3 // global jump to virtual address ENDPROC(cpu_resume_mmu) + .popsection cpu_resume_after_mmu: mov x0, #0 // return zero on success ldp x19, x20, [sp, #16] diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index a2c2986..9807333 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -38,6 +38,12 @@ jiffies = jiffies_64; *(.hyp.text) \ VMLINUX_SYMBOL(__hyp_text_end) = .; +#define IDMAP_TEXT \ + . = ALIGN(SZ_4K); \ + VMLINUX_SYMBOL(__idmap_text_start) = .; \ + *(.idmap.text) \ + VMLINUX_SYMBOL(__idmap_text_end) = .; + /* * The size of the PE/COFF section that covers the kernel image, which * runs from stext to _edata, must be a round multiple of the PE/COFF @@ -95,6 +101,7 @@ SECTIONS SCHED_TEXT LOCK_TEXT HYPERVISOR_TEXT + IDMAP_TEXT *(.fixup) *(.gnu.warning) . = ALIGN(16); @@ -167,11 +174,13 @@ SECTIONS } /* - * The HYP init code can't be more than a page long, + * The HYP init code and ID map text can't be longer than a page each, * and should not cross a page boundary. */ ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K, "HYP init code too big or misaligned") +ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K, + "ID map text too big or misaligned") /* * If padding is applied before .head.text, virt<->phys conversions will fail. -- cgit v0.10.2 From 9acdc2af0c0b836183b7f31f630bbed341a7cf4d Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 1 Jun 2015 13:40:34 +0200 Subject: arm64: drop sleep_idmap_phys and clean up cpu_resume() Two cleanups of the asm function cpu_resume(): - The global variable sleep_idmap_phys always points to idmap_pg_dir, so we can just use that value directly in the CPU resume path. - Unclutter the load of sleep_save_sp::save_ptr_stash_phys. Acked-by: Lorenzo Pieralisi Tested-by: Lorenzo Pieralisi Signed-off-by: Ard Biesheuvel Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index 811e61a..803cfea 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -164,15 +164,12 @@ ENTRY(cpu_resume) #else mov x7, xzr #endif - adrp x0, sleep_save_sp - add x0, x0, #:lo12:sleep_save_sp - ldr x0, [x0, #SLEEP_SAVE_SP_PHYS] + ldr_l x0, sleep_save_sp + SLEEP_SAVE_SP_PHYS ldr x0, [x0, x7, lsl #3] /* load sp from context */ ldr x2, [x0, #CPU_CTX_SP] - adrp x1, sleep_idmap_phys /* load physical address of identity map page table in x1 */ - ldr x1, [x1, #:lo12:sleep_idmap_phys] + adrp x1, idmap_pg_dir mov sp, x2 /* * cpu_do_resume expects x0 to contain context physical address diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index d7daf45..f6073c2 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -118,7 +118,6 @@ int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) } struct sleep_save_sp sleep_save_sp; -phys_addr_t sleep_idmap_phys; static int __init cpu_suspend_init(void) { @@ -132,9 +131,7 @@ static int __init cpu_suspend_init(void) sleep_save_sp.save_ptr_stash = ctx_ptr; sleep_save_sp.save_ptr_stash_phys = virt_to_phys(ctx_ptr); - sleep_idmap_phys = virt_to_phys(idmap_pg_dir); __flush_dcache_area(&sleep_save_sp, sizeof(struct sleep_save_sp)); - __flush_dcache_area(&sleep_idmap_phys, sizeof(sleep_idmap_phys)); return 0; } -- cgit v0.10.2 From 10b48f7ef2a765452a583dbc256ed7da7fe7f1fc Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 1 Jun 2015 10:47:39 +0100 Subject: arm64: insn: Add aarch64_{get,set}_branch_offset In order to deal with branches located in alternate sequences, but pointing to the main kernel text, it is required to extract the relative displacement encoded in the instruction, and to be able to update said instruction with a new offset (once it is known). For this, we introduce three new helpers: - aarch64_insn_is_branch_imm is a predicate indicating if the instruction is an immediate branch - aarch64_get_branch_offset returns a signed value representing the byte offset encoded in a branch instruction - aarch64_set_branch_offset takes an instruction and an offset, and returns the corresponding updated instruction. Acked-by: Will Deacon Signed-off-by: Marc Zyngier Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index f81b328..30e50eb 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -281,6 +281,7 @@ __AARCH64_INSN_FUNCS(ret, 0xFFFFFC1F, 0xD65F0000) #undef __AARCH64_INSN_FUNCS bool aarch64_insn_is_nop(u32 insn); +bool aarch64_insn_is_branch_imm(u32 insn); int aarch64_insn_read(void *addr, u32 *insnp); int aarch64_insn_write(void *addr, u32 insn); @@ -351,6 +352,8 @@ u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst, int shift, enum aarch64_insn_variant variant, enum aarch64_insn_logic_type type); +s32 aarch64_get_branch_offset(u32 insn); +u32 aarch64_set_branch_offset(u32 insn, s32 offset); bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn); diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c index 9249020..dd9671c 100644 --- a/arch/arm64/kernel/insn.c +++ b/arch/arm64/kernel/insn.c @@ -77,6 +77,14 @@ bool __kprobes aarch64_insn_is_nop(u32 insn) } } +bool aarch64_insn_is_branch_imm(u32 insn) +{ + return (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn) || + aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn) || + aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || + aarch64_insn_is_bcond(insn)); +} + static DEFINE_SPINLOCK(patch_lock); static void __kprobes *patch_map(void *addr, int fixmap) @@ -1057,6 +1065,58 @@ u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst, return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); } +/* + * Decode the imm field of a branch, and return the byte offset as a + * signed value (so it can be used when computing a new branch + * target). + */ +s32 aarch64_get_branch_offset(u32 insn) +{ + s32 imm; + + if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) { + imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn); + return (imm << 6) >> 4; + } + + if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || + aarch64_insn_is_bcond(insn)) { + imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn); + return (imm << 13) >> 11; + } + + if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) { + imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn); + return (imm << 18) >> 16; + } + + /* Unhandled instruction */ + BUG(); +} + +/* + * Encode the displacement of a branch in the imm field and return the + * updated instruction. + */ +u32 aarch64_set_branch_offset(u32 insn, s32 offset) +{ + if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) + return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn, + offset >> 2); + + if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) || + aarch64_insn_is_bcond(insn)) + return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn, + offset >> 2); + + if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) + return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn, + offset >> 2); + + /* Unhandled instruction */ + BUG(); +} + bool aarch32_insn_is_wide(u32 insn) { return insn >= 0xe800; -- cgit v0.10.2 From b0dd9c02d476162340ad60fc96befa817fa8fe9f Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 3 Jun 2015 14:36:23 +0100 Subject: arm64: Rework alternate sequence for ARM erratum 845719 The workaround for erratum 845719 is currently using a branch between two alternate sequences, which is quite fragile, and that we are going to break as we rework the alternative code. This patch reworks the workaround to fit in a single alternative sequence. The generated code itself is unchanged. Signed-off-by: Marc Zyngier Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 959fe87..9fe1a85 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -124,21 +124,24 @@ msr sp_el0, x23 #ifdef CONFIG_ARM64_ERRATUM_845719 - alternative_insn \ - "nop", \ - "tbz x22, #4, 1f", \ - ARM64_WORKAROUND_845719 + +#undef SEQUENCE_ORG +#undef SEQUENCE_ALT + #ifdef CONFIG_PID_IN_CONTEXTIDR - alternative_insn \ - "nop; nop", \ - "mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:", \ - ARM64_WORKAROUND_845719 + +#define SEQUENCE_ORG "nop ; nop ; nop" +#define SEQUENCE_ALT "tbz x22, #4, 1f ; mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:" + #else - alternative_insn \ - "nop", \ - "msr contextidr_el1, xzr; 1:", \ - ARM64_WORKAROUND_845719 + +#define SEQUENCE_ORG "nop ; nop" +#define SEQUENCE_ALT "tbz x22, #4, 1f ; msr contextidr_el1, xzr; 1:" + #endif + + alternative_insn SEQUENCE_ORG, SEQUENCE_ALT, ARM64_WORKAROUND_845719 + #endif .endif msr elr_el1, x21 // set up the return data -- cgit v0.10.2 From 7616fc8bcd7ef4975a294337d6cd3007b8c18746 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 1 Jun 2015 10:47:40 +0100 Subject: arm64: alternative: Allow immediate branch as alternative instruction Since all branches are PC-relative on AArch64, these instructions cannot be used as an alternative with the simplistic approach we currently have (the immediate has been computed from the .altinstr_replacement section, and end-up being completely off if the target is outside of the replacement sequence). This patch handles the branch instructions in a different way, using the insn framework to recompute the immediate, and generate the right displacement in the above case. Acked-by: Will Deacon Signed-off-by: Marc Zyngier Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index 28f8365..221b983 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -24,8 +24,13 @@ #include #include #include +#include #include +#define __ALT_PTR(a,f) (u32 *)((void *)&(a)->f + (a)->f) +#define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset) +#define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset) + extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; struct alt_region { @@ -33,13 +38,63 @@ struct alt_region { struct alt_instr *end; }; +/* + * Check if the target PC is within an alternative block. + */ +static bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc) +{ + unsigned long replptr; + + if (kernel_text_address(pc)) + return 1; + + replptr = (unsigned long)ALT_REPL_PTR(alt); + if (pc >= replptr && pc <= (replptr + alt->alt_len)) + return 0; + + /* + * Branching into *another* alternate sequence is doomed, and + * we're not even trying to fix it up. + */ + BUG(); +} + +static u32 get_alt_insn(struct alt_instr *alt, u32 *insnptr, u32 *altinsnptr) +{ + u32 insn; + + insn = le32_to_cpu(*altinsnptr); + + if (aarch64_insn_is_branch_imm(insn)) { + s32 offset = aarch64_get_branch_offset(insn); + unsigned long target; + + target = (unsigned long)altinsnptr + offset; + + /* + * If we're branching inside the alternate sequence, + * do not rewrite the instruction, as it is already + * correct. Otherwise, generate the new instruction. + */ + if (branch_insn_requires_update(alt, target)) { + offset = target - (unsigned long)insnptr; + insn = aarch64_set_branch_offset(insn, offset); + } + } + + return insn; +} + static int __apply_alternatives(void *alt_region) { struct alt_instr *alt; struct alt_region *region = alt_region; - u8 *origptr, *replptr; + u32 *origptr, *replptr; for (alt = region->begin; alt < region->end; alt++) { + u32 insn; + int i, nr_inst; + if (!cpus_have_cap(alt->cpufeature)) continue; @@ -47,11 +102,17 @@ static int __apply_alternatives(void *alt_region) pr_info_once("patching kernel code\n"); - origptr = (u8 *)&alt->orig_offset + alt->orig_offset; - replptr = (u8 *)&alt->alt_offset + alt->alt_offset; - memcpy(origptr, replptr, alt->alt_len); + origptr = ALT_ORIG_PTR(alt); + replptr = ALT_REPL_PTR(alt); + nr_inst = alt->alt_len / sizeof(insn); + + for (i = 0; i < nr_inst; i++) { + insn = get_alt_insn(alt, origptr + i, replptr + i); + *(origptr + i) = cpu_to_le32(insn); + } + flush_icache_range((uintptr_t)origptr, - (uintptr_t)(origptr + alt->alt_len)); + (uintptr_t)(origptr + nr_inst)); } return 0; -- cgit v0.10.2 From 8d883b23aed73cad844ba48051c7e96eddf0f51c Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 1 Jun 2015 10:47:41 +0100 Subject: arm64: alternative: Merge alternative-asm.h into alternative.h asm/alternative-asm.h and asm/alternative.h are extremely similar, and really deserve to live in the same file (as this makes further modufications a bit easier). Fold the content of alternative-asm.h into alternative.h, and update the few users. Acked-by: Will Deacon Signed-off-by: Marc Zyngier Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/alternative-asm.h b/arch/arm64/include/asm/alternative-asm.h deleted file mode 100644 index 919a678..0000000 --- a/arch/arm64/include/asm/alternative-asm.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef __ASM_ALTERNATIVE_ASM_H -#define __ASM_ALTERNATIVE_ASM_H - -#ifdef __ASSEMBLY__ - -.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len - .word \orig_offset - . - .word \alt_offset - . - .hword \feature - .byte \orig_len - .byte \alt_len -.endm - -.macro alternative_insn insn1 insn2 cap -661: \insn1 -662: .pushsection .altinstructions, "a" - altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f - .popsection - .pushsection .altinstr_replacement, "ax" -663: \insn2 -664: .popsection - .if ((664b-663b) != (662b-661b)) - .error "Alternatives instruction length mismatch" - .endif -.endm - -#endif /* __ASSEMBLY__ */ - -#endif /* __ASM_ALTERNATIVE_ASM_H */ diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index d261f01..265b13e 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -1,6 +1,8 @@ #ifndef __ASM_ALTERNATIVE_H #define __ASM_ALTERNATIVE_H +#ifndef __ASSEMBLY__ + #include #include #include @@ -41,4 +43,29 @@ void free_alternatives_memory(void); " .error \"Alternatives instruction length mismatch\"\n\t"\ ".endif\n" +#else + +.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len + .word \orig_offset - . + .word \alt_offset - . + .hword \feature + .byte \orig_len + .byte \alt_len +.endm + +.macro alternative_insn insn1 insn2 cap +661: \insn1 +662: .pushsection .altinstructions, "a" + altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f + .popsection + .pushsection .altinstr_replacement, "ax" +663: \insn2 +664: .popsection + .if ((664b-663b) != (662b-661b)) + .error "Alternatives instruction length mismatch" + .endif +.endm + +#endif /* __ASSEMBLY__ */ + #endif /* __ASM_ALTERNATIVE_H */ diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 9fe1a85..d369ccf 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -21,7 +21,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index f563e9a..bdeb5d3 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include "proc-macros.S" -- cgit v0.10.2 From eb7c11ee3c5ce6c45ac28a5015a8e60ed458b412 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 1 Jun 2015 10:47:42 +0100 Subject: arm64: alternative: Work around .inst assembler bugs AArch64 toolchains suffer from the following bug: $ cat blah.S 1: .inst 0x01020304 .if ((. - 1b) != 4) .error "blah" .endif $ aarch64-linux-gnu-gcc -c blah.S blah.S: Assembler messages: blah.S:3: Error: non-constant expression in ".if" statement which precludes the use of msr_s and co as part of alternatives. We workaround this issue by not directly testing the labels themselves, but by moving the current output pointer by a value that should always be zero. If this value is not null, then we will trigger a backward move, which is expclicitely forbidden. This triggers the error we're after: AS arch/arm64/kvm/hyp.o arch/arm64/kvm/hyp.S: Assembler messages: arch/arm64/kvm/hyp.S:1377: Error: attempt to move .org backwards scripts/Makefile.build:294: recipe for target 'arch/arm64/kvm/hyp.o' failed make[1]: *** [arch/arm64/kvm/hyp.o] Error 1 Makefile:946: recipe for target 'arch/arm64/kvm' failed Not pretty, but at least works on the current toolchains. Acked-by: Will Deacon Signed-off-by: Marc Zyngier Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index 265b13e..c385a0c 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -26,7 +26,20 @@ void free_alternatives_memory(void); " .byte 662b-661b\n" /* source len */ \ " .byte 664f-663f\n" /* replacement len */ -/* alternative assembly primitive: */ +/* + * alternative assembly primitive: + * + * If any of these .org directive fail, it means that insn1 and insn2 + * don't have the same length. This used to be written as + * + * .if ((664b-663b) != (662b-661b)) + * .error "Alternatives instruction length mismatch" + * .endif + * + * but most assemblers die if insn1 or insn2 have a .inst. This should + * be fixed in a binutils release posterior to 2.25.51.0.2 (anything + * containing commit 4e4d08cf7399b606 or c1baaddf8861). + */ #define ALTERNATIVE(oldinstr, newinstr, feature) \ "661:\n\t" \ oldinstr "\n" \ @@ -39,9 +52,8 @@ void free_alternatives_memory(void); newinstr "\n" \ "664:\n\t" \ ".popsection\n\t" \ - ".if ((664b-663b) != (662b-661b))\n\t" \ - " .error \"Alternatives instruction length mismatch\"\n\t"\ - ".endif\n" + ".org . - (664b-663b) + (662b-661b)\n\t" \ + ".org . - (662b-661b) + (664b-663b)\n" #else @@ -61,9 +73,8 @@ void free_alternatives_memory(void); .pushsection .altinstr_replacement, "ax" 663: \insn2 664: .popsection - .if ((664b-663b) != (662b-661b)) - .error "Alternatives instruction length mismatch" - .endif + .org . - (664b-663b) + (662b-661b) + .org . - (662b-661b) + (664b-663b) .endm #endif /* __ASSEMBLY__ */ -- cgit v0.10.2 From 04d7e098f541769721d7511d56aea4b976fd29fd Mon Sep 17 00:00:00 2001 From: Josh Stone Date: Fri, 5 Jun 2015 14:28:03 -0700 Subject: arm64: fix missing syscall trace exit If a syscall is entered without TIF_SYSCALL_TRACE set, then it goes on the fast path. It's then possible to have TIF_SYSCALL_TRACE added in the middle of the syscall, but ret_fast_syscall doesn't check this flag again. This causes a ptrace syscall-exit-stop to be missed. For instance, from a PTRACE_EVENT_FORK reported during do_fork, the tracer might resume with PTRACE_SYSCALL, setting TIF_SYSCALL_TRACE. Now the completion of the fork should have a syscall-exit-stop. Russell King fixed this on arm by re-checking _TIF_SYSCALL_WORK in the fast exit path. Do the same on arm64. Reviewed-by: Will Deacon Cc: Russell King Signed-off-by: Josh Stone Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index d369ccf..0eeb1b9 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -611,11 +611,16 @@ ENDPROC(cpu_switch_to) */ ret_fast_syscall: disable_irq // disable interrupts - ldr x1, [tsk, #TI_FLAGS] + ldr x1, [tsk, #TI_FLAGS] // re-check for syscall tracing + and x2, x1, #_TIF_SYSCALL_WORK + cbnz x2, ret_fast_syscall_trace and x2, x1, #_TIF_WORK_MASK cbnz x2, fast_work_pending enable_step_tsk x1, x2 kernel_exit 0, ret = 1 +ret_fast_syscall_trace: + enable_irq // enable interrupts + b __sys_trace_return /* * Ok, we need to do extra processing, enter the slow path. -- cgit v0.10.2 From 6eb6c80187c55b7f95683bc6502dccac54b95b92 Mon Sep 17 00:00:00 2001 From: Janet Liu Date: Thu, 11 Jun 2015 12:04:32 +0800 Subject: arm64: kernel thread don't need to save fpsimd context. kernel thread's default fpsimd state is zero. When fork a thread, if parent is kernel thread, and save hardware context to parent's fpsimd state, but this hardware context is user process's context, because kernel thread don't use fpsimd, it will not introduce issue, it add a little cost. Signed-off-by: Janet Liu Signed-off-by: Chunyan Zhang Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 369f485..223b093 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -233,7 +233,8 @@ void release_thread(struct task_struct *dead_task) int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) { - fpsimd_preserve_current_state(); + if (current->mm) + fpsimd_preserve_current_state(); *dst = *src; return 0; } -- cgit v0.10.2 From 32365e64a20edcc783137ad17fdd951ab814a2fe Mon Sep 17 00:00:00 2001 From: Janet Liu Date: Thu, 11 Jun 2015 12:02:45 +0800 Subject: arm64: fix bug for reloading FPSIMD state after CPU hotplug. Now FPSIMD don't handle HOTPLUG_CPU. This introduces bug after cpu down/up process. After cpu down/up process, the FPSMID hardware register is default value, not any process's fpsimd context. when CPU_DEAD set cpu's fpsimd_state to NULL, it will force to load the fpsimd context for the thread, to avoid the chance to skip to load the context. If process A is the last user process on CPU N before cpu down, and the first user process on the same CPU N after cpu up, A's fpsimd_state.cpu is the current cpu id, and per_cpu(fpsimd_last_state) points A's fpsimd_state, so kernel will not reload the context during it return to user space. Signed-off-by: Janet Liu Signed-off-by: Xiongshan An Signed-off-by: Chunyan Zhang [catalin.marinas@arm.com: some mostly cosmetic clean-ups] Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index 3dca156..44d6f75 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -17,6 +17,7 @@ * along with this program. If not, see . */ +#include #include #include #include @@ -296,6 +297,35 @@ static void fpsimd_pm_init(void) static inline void fpsimd_pm_init(void) { } #endif /* CONFIG_CPU_PM */ +#ifdef CONFIG_HOTPLUG_CPU +static int fpsimd_cpu_hotplug_notifier(struct notifier_block *nfb, + unsigned long action, + void *hcpu) +{ + unsigned int cpu = (long)hcpu; + + switch (action) { + case CPU_DEAD: + case CPU_DEAD_FROZEN: + per_cpu(fpsimd_last_state, cpu) = NULL; + break; + } + return NOTIFY_OK; +} + +static struct notifier_block fpsimd_cpu_hotplug_notifier_block = { + .notifier_call = fpsimd_cpu_hotplug_notifier, +}; + +static inline void fpsimd_hotplug_init(void) +{ + register_cpu_notifier(&fpsimd_cpu_hotplug_notifier_block); +} + +#else +static inline void fpsimd_hotplug_init(void) { } +#endif + /* * FP/SIMD support code initialisation. */ @@ -315,6 +345,7 @@ static int __init fpsimd_init(void) elf_hwcap |= HWCAP_ASIMD; fpsimd_pm_init(); + fpsimd_hotplug_init(); return 0; } -- cgit v0.10.2 From 73bf8412e4f24b114c853012663fff4d3cde06a2 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 11 Jun 2015 18:16:47 +0100 Subject: arm64: psci: fix !CONFIG_HOTPLUG_CPU build warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building without CONFIG_HOTPLUG_CPU, GCC complains (rightly) that psci_tos_resident_on is unused: arch/arm64/kernel/psci.c:61:13: warning: ‘psci_tos_resident_on’ defined but not used [-Wunused-function] static bool psci_tos_resident_on(int cpu) As it's only ever used when CONFIG_HOTPLUG_CPU is selected, let's move it into the existing ifdef. Signed-off-by: Will Deacon [Mark: write commit message] Signed-off-by: Mark Rutland Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index cba05d9..2052137 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -58,11 +58,6 @@ static bool psci_power_state_is_valid(u32 state) */ static int resident_cpu = -1; -static bool psci_tos_resident_on(int cpu) -{ - return cpu == resident_cpu; -} - struct psci_operations { int (*cpu_suspend)(u32 state, unsigned long entry_point); int (*cpu_off)(u32 state); @@ -494,6 +489,11 @@ static int cpu_psci_cpu_boot(unsigned int cpu) } #ifdef CONFIG_HOTPLUG_CPU +static bool psci_tos_resident_on(int cpu) +{ + return cpu == resident_cpu; +} + static int cpu_psci_cpu_disable(unsigned int cpu) { /* Fail early if we don't have CPU_OFF support */ -- cgit v0.10.2 From 94a9e04aa16abd1194d9b4158c618ba87f5d01e6 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 12 Jun 2015 12:06:36 +0100 Subject: arm64: alternative: Introduce feature for GICv3 CPU interface Add a new item to the feature set (ARM64_HAS_SYSREG_GIC_CPUIF) to indicate that we have a system register GIC CPU interface This will help KVM switching to alternative instruction patching. Reviewed-by: Andre Przywara Reviewed-by: Christoffer Dall Acked-by: Will Deacon Signed-off-by: Marc Zyngier Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 82cb9f9..c104421 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -24,8 +24,9 @@ #define ARM64_WORKAROUND_CLEAN_CACHE 0 #define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE 1 #define ARM64_WORKAROUND_845719 2 +#define ARM64_HAS_SYSREG_GIC_CPUIF 3 -#define ARM64_NCAPS 3 +#define ARM64_NCAPS 4 #ifndef __ASSEMBLY__ @@ -38,6 +39,11 @@ struct arm64_cpu_capabilities { u32 midr_model; u32 midr_range_min, midr_range_max; }; + + struct { /* Feature register checking */ + u64 register_mask; + u64 register_value; + }; }; }; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 3d9967e..5ad86ce 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -22,7 +22,23 @@ #include #include +static bool +has_id_aa64pfr0_feature(const struct arm64_cpu_capabilities *entry) +{ + u64 val; + + val = read_cpuid(id_aa64pfr0_el1); + return (val & entry->register_mask) == entry->register_value; +} + static const struct arm64_cpu_capabilities arm64_features[] = { + { + .desc = "GIC system register CPU interface", + .capability = ARM64_HAS_SYSREG_GIC_CPUIF, + .matches = has_id_aa64pfr0_feature, + .register_mask = (0xf << 24), + .register_value = (1 << 24), + }, {}, }; -- cgit v0.10.2 From 8a14849b4a355278f0b7baf6e2da7dc7144a23e8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 12 Jun 2015 12:06:37 +0100 Subject: arm64: KVM: Switch vgic save/restore to alternative_insn So far, we configured the world-switch by having a small array of pointers to the save and restore functions, depending on the GIC used on the platform. Loading these values each time is a bit silly (they never change), and it makes sense to rely on the instruction patching instead. This leads to a nice cleanup of the code. Acked-by: Will Deacon Reviewed-by: Christoffer Dall Signed-off-by: Marc Zyngier Signed-off-by: Catalin Marinas diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index d71607c..e896d2c 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h @@ -218,11 +218,6 @@ static inline int kvm_arch_dev_ioctl_check_extension(long ext) return 0; } -static inline void vgic_arch_setup(const struct vgic_params *vgic) -{ - BUG_ON(vgic->type != VGIC_V2); -} - int kvm_perf_init(void); int kvm_perf_teardown(void); diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 4f7310f..d13bc5e 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -132,11 +132,6 @@ extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); extern u64 __vgic_v3_get_ich_vtr_el2(void); -extern char __save_vgic_v2_state[]; -extern char __restore_vgic_v2_state[]; -extern char __save_vgic_v3_state[]; -extern char __restore_vgic_v3_state[]; - #endif #endif /* __ARM_KVM_ASM_H__ */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index f0f58c9..2709db2 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -221,29 +221,6 @@ struct vgic_sr_vectors { void *restore_vgic; }; -static inline void vgic_arch_setup(const struct vgic_params *vgic) -{ - extern struct vgic_sr_vectors __vgic_sr_vectors; - - switch(vgic->type) - { - case VGIC_V2: - __vgic_sr_vectors.save_vgic = __save_vgic_v2_state; - __vgic_sr_vectors.restore_vgic = __restore_vgic_v2_state; - break; - -#ifdef CONFIG_ARM_GIC_V3 - case VGIC_V3: - __vgic_sr_vectors.save_vgic = __save_vgic_v3_state; - __vgic_sr_vectors.restore_vgic = __restore_vgic_v3_state; - break; -#endif - - default: - BUG(); - } -} - static inline void kvm_arch_hardware_disable(void) {} static inline void kvm_arch_hardware_unsetup(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index da675cc..c99701a 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -127,7 +127,6 @@ int main(void) DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu)); DEFINE(VGIC_SAVE_FN, offsetof(struct vgic_sr_vectors, save_vgic)); DEFINE(VGIC_RESTORE_FN, offsetof(struct vgic_sr_vectors, restore_vgic)); - DEFINE(VGIC_SR_VECTOR_SZ, sizeof(struct vgic_sr_vectors)); DEFINE(VGIC_V2_CPU_HCR, offsetof(struct vgic_cpu, vgic_v2.vgic_hcr)); DEFINE(VGIC_V2_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v2.vgic_vmcr)); DEFINE(VGIC_V2_CPU_MISR, offsetof(struct vgic_cpu, vgic_v2.vgic_misr)); diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S index 5befd01..f1f6d90 100644 --- a/arch/arm64/kvm/hyp.S +++ b/arch/arm64/kvm/hyp.S @@ -17,8 +17,10 @@ #include +#include #include #include +#include #include #include #include @@ -808,10 +810,7 @@ * Call into the vgic backend for state saving */ .macro save_vgic_state - adr x24, __vgic_sr_vectors - ldr x24, [x24, VGIC_SAVE_FN] - kern_hyp_va x24 - blr x24 + alternative_insn "bl __save_vgic_v2_state", "bl __save_vgic_v3_state", ARM64_HAS_SYSREG_GIC_CPUIF mrs x24, hcr_el2 mov x25, #HCR_INT_OVERRIDE neg x25, x25 @@ -828,10 +827,7 @@ orr x24, x24, #HCR_INT_OVERRIDE orr x24, x24, x25 msr hcr_el2, x24 - adr x24, __vgic_sr_vectors - ldr x24, [x24, #VGIC_RESTORE_FN] - kern_hyp_va x24 - blr x24 + alternative_insn "bl __restore_vgic_v2_state", "bl __restore_vgic_v3_state", ARM64_HAS_SYSREG_GIC_CPUIF .endm .macro save_timer_state @@ -1062,12 +1058,6 @@ ENTRY(__kvm_flush_vm_context) ret ENDPROC(__kvm_flush_vm_context) - // struct vgic_sr_vectors __vgi_sr_vectors; - .align 3 -ENTRY(__vgic_sr_vectors) - .skip VGIC_SR_VECTOR_SZ -ENDPROC(__vgic_sr_vectors) - __kvm_hyp_panic: // Guess the context by looking at VTTBR: // If zero, then we're already a host. diff --git a/virt/kvm/arm/vgic.c b/virt/kvm/arm/vgic.c index 78fb820..003ba74 100644 --- a/virt/kvm/arm/vgic.c +++ b/virt/kvm/arm/vgic.c @@ -2128,9 +2128,6 @@ int kvm_vgic_hyp_init(void) goto out_free_irq; } - /* Callback into for arch code for setup */ - vgic_arch_setup(vgic); - on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1); return 0; -- cgit v0.10.2 From 565630d503ef24e44c252bed55571b3a0d68455f Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 12 Jun 2015 11:24:41 +0100 Subject: arm64: Do not attempt to use init_mm in reset_context() After secondary CPU boot or hotplug, the active_mm of the idle thread is &init_mm. The init_mm.pgd (swapper_pg_dir) is only meant for TTBR1_EL1 and must not be set in TTBR0_EL1. Since when active_mm == &init_mm the TTBR0_EL1 is already set to the reserved value, there is no need to perform any context reset. Signed-off-by: Catalin Marinas Cc: diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index baa758d..76c1e6c 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -92,6 +92,14 @@ static void reset_context(void *info) unsigned int cpu = smp_processor_id(); struct mm_struct *mm = current->active_mm; + /* + * current->active_mm could be init_mm for the idle thread immediately + * after secondary CPU boot or hotplug. TTBR0_EL1 is already set to + * the reserved value, so no need to reset any context. + */ + if (mm == &init_mm) + return; + smp_rmb(); asid = cpu_last_asid + cpu; -- cgit v0.10.2 From c7d6b573fe55fdccd60bfe082dda8cace9bcaa3f Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Fri, 12 Jun 2015 15:25:04 +0100 Subject: arm64: mm: remove reference to tlb.S from comment block tlb.S has been removed since fa48e6f "arm64: mm: Optimise tlb flush logic where we have >4K granule", so align comment with that. Signed-off-by: Vladimir Murzin Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index c3bb05b..934815d 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -28,8 +28,6 @@ * TLB Management * ============== * - * The arch/arm64/mm/tlb.S files implement these methods. - * * The TLB specific code is expected to perform whatever tests it needs * to determine if it should invalidate the TLB for each call. Start * addresses are inclusive and end addresses are exclusive; it is safe to -- cgit v0.10.2 From 822bf4833ecc8ea63c69f3ed894c13b4509c9e85 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 15 Jun 2015 15:02:44 +0100 Subject: arm64: defconfig: enable memtest The kernel memtest utility is incredibly useful for detecting memory problems, but sadly isn't in defconfig. The memtest itself is only run when the user has explicitly passed a memtest option on the kernel command line, so simply enabling the option should not have a negative impact. Signed-off-by: Mark Rutland Cc: Vladimir Murzin Cc: Will Deacon Signed-off-by: Catalin Marinas diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 2ed7449..daefbf0 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -180,6 +180,7 @@ CONFIG_LOCKUP_DETECTOR=y # CONFIG_SCHED_DEBUG is not set # CONFIG_DEBUG_PREEMPT is not set # CONFIG_FTRACE is not set +CONFIG_MEMTEST=y CONFIG_SECURITY=y CONFIG_CRYPTO_ANSI_CPRNG=y CONFIG_ARM64_CRYPTO=y -- cgit v0.10.2 From 46b0567c851cf85d6ba6f23eef385ec9111d09bc Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 15 Jun 2015 16:40:27 +0100 Subject: arm64: entry: fix context tracking for el0_sp_pc Commit 6c81fe7925cc4c42 ("arm64: enable context tracking") did not update el0_sp_pc to use ct_user_exit, but this appears to have been unintentional. In commit 6ab6463aeb5fbc75 ("arm64: adjust el0_sync so that a function can be called") we made x0 available, and in the return to userspace we call ct_user_enter in the kernel_exit macro. Due to this, we currently don't correctly inform RCU of the user->kernel transition, and may erroneously account for time spent in the kernel as if we were in an extended quiescent state when CONFIG_CONTEXT_TRACKING is enabled. As we do record the kernel->user transition, a userspace application making accesses from an unaligned stack pointer can demonstrate the imbalance, provoking the following warning: ------------[ cut here ]------------ WARNING: CPU: 2 PID: 3660 at kernel/context_tracking.c:75 context_tracking_enter+0xd8/0xe4() Modules linked in: CPU: 2 PID: 3660 Comm: a.out Not tainted 4.1.0-rc7+ #8 Hardware name: ARM Juno development board (r0) (DT) Call trace: [] dump_backtrace+0x0/0x124 [] show_stack+0x10/0x1c [] dump_stack+0x84/0xc8 [] warn_slowpath_common+0x98/0xd0 [] warn_slowpath_null+0x14/0x20 [] context_tracking_enter+0xd4/0xe4 [] preempt_schedule_irq+0xd4/0x114 [] el1_preempt+0x4/0x28 [] exit_files+0x38/0x4c [] do_exit+0x430/0x978 [] do_group_exit+0x40/0xd4 [] get_signal+0x23c/0x4f4 [] do_signal+0x1ac/0x518 [] do_notify_resume+0x5c/0x68 ---[ end trace 963c192600337066 ]--- This patch adds the missing ct_user_exit to the el0_sp_pc entry path, correcting the context tracking for this case. Signed-off-by: Mark Rutland Acked-by: Will Deacon Fixes: 6c81fe7925cc ("arm64: enable context tracking") Cc: # v3.17+ Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 0eeb1b9..a7691a3 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -520,6 +520,7 @@ el0_sp_pc: mrs x26, far_el1 // enable interrupts before calling the main handler enable_dbg_and_irq + ct_user_exit mov x0, x26 mov x1, x25 mov x2, sp -- cgit v0.10.2 From b9bcc919931611498e856eae9bf66337330d04cc Mon Sep 17 00:00:00 2001 From: Dave P Martin Date: Tue, 16 Jun 2015 17:38:47 +0100 Subject: arm64: mm: Fix freeing of the wrong memmap entries with !SPARSEMEM_VMEMMAP The memmap freeing code in free_unused_memmap() computes the end of each memblock by adding the memblock size onto the base. However, if SPARSEMEM is enabled then the value (start) used for the base may already have been rounded downwards to work out which memmap entries to free after the previous memblock. This may cause memmap entries that are in use to get freed. In general, you're not likely to hit this problem unless there are at least 2 memblocks and one of them is not aligned to a sparsemem section boundary. Note that carve-outs can increase the number of memblocks by splitting the regions listed in the device tree. This problem doesn't occur with SPARSEMEM_VMEMMAP, because the vmemmap code deals with freeing the unused regions of the memmap instead of requiring the arch code to do it. This patch gets the memblock base out of the memblock directly when computing the block end address to ensure the correct value is used. Signed-off-by: Dave Martin Cc: Signed-off-by: Catalin Marinas diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 597831b..ad87ce8 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -262,7 +262,7 @@ static void __init free_unused_memmap(void) * memmap entries are valid from the bank end aligned to * MAX_ORDER_NR_PAGES. */ - prev_end = ALIGN(start + __phys_to_pfn(reg->size), + prev_end = ALIGN(__phys_to_pfn(reg->base + reg->size), MAX_ORDER_NR_PAGES); } -- cgit v0.10.2 From 4e2ee96a63d434177ad1785208fe06858ebfe739 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 15 Jun 2015 17:30:17 +0100 Subject: arm64: compat: print compat_sp instead of sp We check against compat_sp, but print out arm64's sp - fix it. Signed-off-by: Vladimir Murzin Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c index d26fcd4..1670f15 100644 --- a/arch/arm64/kernel/signal32.c +++ b/arch/arm64/kernel/signal32.c @@ -370,7 +370,7 @@ badframe: if (show_unhandled_signals) pr_info_ratelimited("%s[%d]: bad frame in %s: pc=%08llx sp=%08llx\n", current->comm, task_pid_nr(current), __func__, - regs->pc, regs->sp); + regs->pc, regs->compat_sp); force_sig(SIGSEGV, current); return 0; } @@ -407,7 +407,7 @@ badframe: if (show_unhandled_signals) pr_info_ratelimited("%s[%d]: bad frame in %s: pc=%08llx sp=%08llx\n", current->comm, task_pid_nr(current), __func__, - regs->pc, regs->sp); + regs->pc, regs->compat_sp); force_sig(SIGSEGV, current); return 0; } -- cgit v0.10.2 From af391b15f7b56ce19f52862d36595637dd42b575 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Thu, 18 Jun 2015 15:41:32 +0100 Subject: arm64: kernel: rename __cpu_suspend to keep it aligned with arm This patch renames __cpu_suspend to cpu_suspend so that it's aligned with ARM32. It also removes the redundant wrapper created. This is in preparation to implement generic PSCI system suspend using the cpu_{suspend,resume} which now has the same interface on both ARM and ARM64. Cc: Mark Rutland Reviewed-by: Lorenzo Pieralisi Reviewed-by: Ashwin Chaugule Signed-off-by: Sudeep Holla Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/cpuidle.h b/arch/arm64/include/asm/cpuidle.h index 141b2fc..0f74f05 100644 --- a/arch/arm64/include/asm/cpuidle.h +++ b/arch/arm64/include/asm/cpuidle.h @@ -5,20 +5,16 @@ #ifdef CONFIG_CPU_IDLE extern int arm_cpuidle_init(unsigned int cpu); -extern int cpu_suspend(unsigned long arg); +extern int arm_cpuidle_suspend(int index); #else static inline int arm_cpuidle_init(unsigned int cpu) { return -EOPNOTSUPP; } -static inline int cpu_suspend(unsigned long arg) +static inline int arm_cpuidle_suspend(int index) { return -EOPNOTSUPP; } #endif -static inline int arm_cpuidle_suspend(int index) -{ - return cpu_suspend(index); -} #endif diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h index 003802f..59a5b0f1 100644 --- a/arch/arm64/include/asm/suspend.h +++ b/arch/arm64/include/asm/suspend.h @@ -21,6 +21,6 @@ struct sleep_save_sp { phys_addr_t save_ptr_stash_phys; }; -extern int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)); +extern int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)); extern void cpu_resume(void); #endif diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c index f8aa169..7ce589c 100644 --- a/arch/arm64/kernel/cpuidle.c +++ b/arch/arm64/kernel/cpuidle.c @@ -32,7 +32,7 @@ int arm_cpuidle_init(unsigned int cpu) * Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU * operations back-end error code otherwise. */ -int cpu_suspend(unsigned long arg) +int arm_cpuidle_suspend(int index) { int cpu = smp_processor_id(); @@ -42,5 +42,5 @@ int cpu_suspend(unsigned long arg) */ if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend) return -EOPNOTSUPP; - return cpu_ops[cpu]->cpu_suspend(arg); + return cpu_ops[cpu]->cpu_suspend(index); } diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index 2052137..869f202 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -574,7 +574,7 @@ static int __maybe_unused cpu_psci_cpu_suspend(unsigned long index) if (!psci_power_state_loses_context(state[index - 1])) ret = psci_ops.cpu_suspend(state[index - 1], 0); else - ret = __cpu_suspend(index, psci_suspend_finisher); + ret = cpu_suspend(index, psci_suspend_finisher); return ret; } diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index f6073c2..8297d50 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -51,13 +51,13 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) } /* - * __cpu_suspend + * cpu_suspend * * arg: argument to pass to the finisher function * fn: finisher function pointer * */ -int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) +int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) { struct mm_struct *mm = current->active_mm; int ret; @@ -82,7 +82,7 @@ int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) * We are resuming from reset with TTBR0_EL1 set to the * idmap to enable the MMU; restore the active_mm mappings in * TTBR0_EL1 unless the active_mm == &init_mm, in which case - * the thread entered __cpu_suspend with TTBR0_EL1 set to + * the thread entered cpu_suspend with TTBR0_EL1 set to * reserved TTBR0 page tables and should be restored as such. */ if (mm == &init_mm) -- cgit v0.10.2 From 6f1a6ae87c0c60d7c462ef8fd071f291aa7a9abb Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 19 Jun 2015 13:56:33 +0100 Subject: arm64: vdso: work-around broken ELF toolchains in Makefile When building the kernel with a bare-metal (ELF) toolchain, the -shared option may not be passed down to collect2, resulting in silent corruption of the vDSO image (in particular, the DYNAMIC section is omitted). The effect of this corruption is that the dynamic linker fails to find the vDSO symbols and libc is instead used for the syscalls that we intended to optimise (e.g. gettimeofday). Functionally, there is no issue as the sigreturn trampoline is still intact and located by the kernel. This patch fixes the problem by explicitly passing -shared to the linker when building the vDSO. Cc: Reported-by: Szabolcs Nagy Reported-by: James Greenlaigh Signed-off-by: Will Deacon Signed-off-by: Catalin Marinas diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile index ff3bdde..f6fe17d 100644 --- a/arch/arm64/kernel/vdso/Makefile +++ b/arch/arm64/kernel/vdso/Makefile @@ -15,6 +15,10 @@ ccflags-y := -shared -fno-common -fno-builtin ccflags-y += -nostdlib -Wl,-soname=linux-vdso.so.1 \ $(call cc-ldoption, -Wl$(comma)--hash-style=sysv) +# Workaround for bare-metal (ELF) toolchains that neglect to pass -shared +# down to collect2, resulting in silent corruption of the vDSO image. +ccflags-y += -Wl,-shared + obj-y += vdso.o extra-y += vdso.lds vdso-offsets.h CPPFLAGS_vdso.lds += -P -C -U$(ARCH) -- cgit v0.10.2 From 9e793ab84ed482047f7226595313f0f3a0aa6854 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Fri, 19 Jun 2015 15:28:16 +0100 Subject: arm64: show unhandled SP/PC alignment faults Report unhandled SP/PC alignment faults if the show_unhandled_signals variable is set (via /proc/sys/debug/exception-trace). Signed-off-by: Vladimir Murzin Signed-off-by: Catalin Marinas diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 96da131..e8c608c 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -478,12 +478,19 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr, struct pt_regs *regs) { struct siginfo info; + struct task_struct *tsk = current; + + if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS)) + pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n", + tsk->comm, task_pid_nr(tsk), + esr_get_class_string(esr), (void *)regs->pc, + (void *)regs->sp); info.si_signo = SIGBUS; info.si_errno = 0; info.si_code = BUS_ADRALN; info.si_addr = (void __user *)addr; - arm64_notify_die("", regs, &info, esr); + arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr); } static struct fault_info debug_fault_info[] = { -- cgit v0.10.2 From 86dca36e6ba019650a94cadf922ea3d06dec0182 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Fri, 19 Jun 2015 15:28:03 +0100 Subject: arm64: use private ratelimit state along with show_unhandled_signals printk_ratelimit() shares the ratelimiting state with other callers what may lead to scenarios where at the time we want to print out debug information we already limited, so nothing appears in the dmesg - this makes exception-trace quite poor helper in debugging. Additionally, we have imbalance with some messages limited with global ratelimit state and other messages limited with their private state defined via pr_*_ratelimited(). To address this inconsistency show_unhandled_signals_ratelimited() macro is introduced and caller sites are converted to use it. Signed-off-by: Vladimir Murzin Signed-off-by: Catalin Marinas diff --git a/arch/arm64/include/asm/system_misc.h b/arch/arm64/include/asm/system_misc.h index 659fbf5..57f110b 100644 --- a/arch/arm64/include/asm/system_misc.h +++ b/arch/arm64/include/asm/system_misc.h @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include struct pt_regs; @@ -43,6 +45,17 @@ extern void __show_regs(struct pt_regs *); extern void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); +#define show_unhandled_signals_ratelimited() \ +({ \ + static DEFINE_RATELIMIT_STATE(_rs, \ + DEFAULT_RATELIMIT_INTERVAL, \ + DEFAULT_RATELIMIT_BURST); \ + bool __show_ratelimited = false; \ + if (show_unhandled_signals && __ratelimit(&_rs)) \ + __show_ratelimited = true; \ + __show_ratelimited; \ +}) + #define UDBG_UNDEFINED (1 << 0) #define UDBG_SYSCALL (1 << 1) #define UDBG_BADABORT (1 << 2) diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 1ef2940..a12251c 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -335,8 +335,7 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs) if (call_undef_hook(regs) == 0) return; - if (show_unhandled_signals && unhandled_signal(current, SIGILL) && - printk_ratelimit()) { + if (show_unhandled_signals_ratelimited() && unhandled_signal(current, SIGILL)) { pr_info("%s[%d]: undefined instruction: pc=%p\n", current->comm, task_pid_nr(current), pc); dump_instr(KERN_INFO, regs); @@ -363,7 +362,7 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs) } #endif - if (show_unhandled_signals && printk_ratelimit()) { + if (show_unhandled_signals_ratelimited()) { pr_info("%s[%d]: syscall %d\n", current->comm, task_pid_nr(current), (int)regs->syscallno); dump_instr("", regs); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index e8c608c..66bd92a 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -115,8 +115,7 @@ static void __do_user_fault(struct task_struct *tsk, unsigned long addr, { struct siginfo si; - if (show_unhandled_signals && unhandled_signal(tsk, sig) && - printk_ratelimit()) { + if (show_unhandled_signals_ratelimited() && unhandled_signal(tsk, sig)) { pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x\n", tsk->comm, task_pid_nr(tsk), fault_name(esr), sig, addr, esr); -- cgit v0.10.2